{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "# -*- coding: utf-8 -*-\n",
    "import random\n",
    "import numpy as np\n",
    "import logging\n",
    "\n",
    "import tensorflow as tf\n",
    "from keras import losses\n",
    "\n",
    "import gameMetaSquares as game\n",
    "import agent\n",
    "import MCTS\n",
    "from collections import deque\n",
    "\n",
    "import itertools\n",
    "import copy\n",
    "\n",
    "from keras.models import clone_model\n",
    "\n",
    "\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import sys\n",
    "\n",
    "from utils import setup_logger\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<module 'MCTS' from 'MCTS.pyc'>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "reload(game)\n",
    "reload(agent)\n",
    "reload(MCTS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BEST PLAYER VERSION 0\n",
      "BEST PLAYER VERSION 0\n",
      "{'loss': [4.4079586791992185], 'val_loss': [4.3660472679138183]}\n",
      "('BEST LOSSES ', 1.1579551497296241, ' ', 3.2825726869831118)\n",
      "('CURRENT LOSSES ', 1.0076848387701627, ' ', 3.1711791036069976)\n",
      "{'drawn': 0, 'current_player': 12, 'best_player': 8}\n",
      "()\n",
      "BEST PLAYER VERSION 1\n",
      "BEST PLAYER VERSION 1\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-3-8e6b1773e3ca>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     76\u001b[0m                 \u001b[0maction\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpi\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbest_player\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mact\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     77\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 78\u001b[0;31m                 \u001b[0maction\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpi\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbest_player\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mact\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     79\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     80\u001b[0m             \u001b[0mlogger_main\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'action: %d'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maction\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/davidfoster/Git/AGI/scripts/agent.pyc\u001b[0m in \u001b[0;36mact\u001b[0;34m(self, state, tau)\u001b[0m\n\u001b[1;32m    180\u001b[0m                 \u001b[0;32mfor\u001b[0m \u001b[0msim\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mMCTSsimulations\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    181\u001b[0m                         \u001b[0mlogger_agent\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'-----STARTING SIMULATION %d ----'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msim\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 182\u001b[0;31m                         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msimulate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    183\u001b[0m                         \u001b[0mlogger_agent\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'-----FINISHED SIMULATION %d ----'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msim\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    184\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/davidfoster/Git/AGI/scripts/agent.pyc\u001b[0m in \u001b[0;36msimulate\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    138\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    139\u001b[0m                 \u001b[0;31m#the reward of 1 is for the player BEFORE the leaf node\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 140\u001b[0;31m                 \u001b[0mleaf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreward\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbreadcrumbs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmcts\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_moveToLeaf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    141\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    142\u001b[0m                 \u001b[0mlogger_agent\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'LEAF...%s'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mleaf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mboard\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/davidfoster/Git/AGI/scripts/MCTS.pyc\u001b[0m in \u001b[0;36m_moveToLeaf\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     90\u001b[0m \t\t\t\tlogger_mcts.info('action: %d... N = %d, P = %f, nu = %f, adjP = %f, W = %f, Q = %f, U = %f, -Q+U = %f'\n\u001b[1;32m     91\u001b[0m                                         \u001b[0;34m,\u001b[0m \u001b[0maction\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnextNode\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstats\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'N'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mround\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnextNode\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstats\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'P'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mround\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnu\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0midx\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mepsilon\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mnextNode\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstats\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'P'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mepsilon\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mnu\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0midx\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 92\u001b[0;31m \t\t\t\t\t, round(nextNode.stats['W'],5), round(Q,5), round(U,5), round(-Q+U,5))\n\u001b[0m\u001b[1;32m     93\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     94\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/Cellar/python/2.7.13/Frameworks/Python.framework/Versions/2.7/lib/python2.7/logging/__init__.pyc\u001b[0m in \u001b[0;36minfo\u001b[0;34m(self, msg, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1165\u001b[0m         \"\"\"\n\u001b[1;32m   1166\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0misEnabledFor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mINFO\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1167\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_log\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mINFO\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmsg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1168\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1169\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mwarning\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmsg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/Cellar/python/2.7.13/Frameworks/Python.framework/Versions/2.7/lib/python2.7/logging/__init__.pyc\u001b[0m in \u001b[0;36m_log\u001b[0;34m(self, level, msg, args, exc_info, extra)\u001b[0m\n\u001b[1;32m   1275\u001b[0m             \u001b[0;31m#IronPython can use logging.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1276\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1277\u001b[0;31m                 \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlno\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfunc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfindCaller\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1278\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1279\u001b[0m                 \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlno\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfunc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"(unknown file)\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"(unknown function)\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/Cellar/python/2.7.13/Frameworks/Python.framework/Versions/2.7/lib/python2.7/logging/__init__.pyc\u001b[0m in \u001b[0;36mfindCaller\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1248\u001b[0m                 \u001b[0mf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf_back\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1249\u001b[0m                 \u001b[0;32mcontinue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1250\u001b[0;31m             \u001b[0mrv\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mco\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mco_filename\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf_lineno\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mco\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mco_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1251\u001b[0m             \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1252\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mrv\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "sess = tf.Session()\n",
    "\n",
    "logger_main = setup_logger('logger_main', 'logger_main.log')\n",
    "\n",
    "logger_memory = setup_logger('logger_memory', 'logger_memory.log')\n",
    "\n",
    "logger_main.disabled = False\n",
    "logger_memory.disabled = False\n",
    "\n",
    "logger_main.info('=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*')\n",
    "logger_main.info('=*=*=*=*=*=.      NEW LOG.      =*=*=*=*=*')\n",
    "logger_main.info('=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*')\n",
    "\n",
    "\n",
    "EPISODES = 1\n",
    "BATCH_SIZE = 500\n",
    "EVAL_EPISODES = 20\n",
    "MCTSsimulations = 100\n",
    "\n",
    "MEMORY_SIZE = 1920\n",
    "\n",
    "env = game.make('XO', MEMORY_SIZE)\n",
    "\n",
    "pieces = {'1':'X', '0': '-', '-1':'O'}\n",
    "\n",
    "state_size = len(env.gameState.binary())\n",
    "action_size = len(env.actionSpace)\n",
    "\n",
    "current_player = agent.Agent(state_size, action_size, MCTSsimulations)\n",
    "best_player = agent.Agent(state_size, action_size, MCTSsimulations)  #100\n",
    "\n",
    "best_players = [best_player]\n",
    "best_player_version = 0\n",
    "\n",
    "##### SELF PLAY\n",
    "\n",
    "while 1:\n",
    "\n",
    "    #current_player = player_list[len(player_list) - 1]\n",
    "    #best_player = player_list[best_player_num]\n",
    "\n",
    "    logger_main.info('BEST PLAYER VERSION: %d', best_player_version)\n",
    "    print('BEST PLAYER VERSION ' + str(best_player_version))\n",
    "\n",
    "    scores = {\"best_player\":0, \"drawn\": 0, \"current_player\":0}\n",
    "\n",
    "    for e in range(EPISODES):\n",
    "        logger_main.info('====================')\n",
    "        logger_main.info('EPISODE %d OF %d', e+1, EPISODES)\n",
    "        logger_main.info('====================')\n",
    "        \n",
    "#       print('EPISODE ' + str(e+1))\n",
    "\n",
    "        state = env.reset()\n",
    "        done = 0\n",
    "        turn = 0\n",
    "#         if best_player.mcts == None:\n",
    "#             print('BEST PLAYER MCTS: ' + '0')\n",
    "#         else:\n",
    "#             print('BEST PLAYER MCTS: ' + str(len(best_player.mcts)))\n",
    "        \n",
    "#         if best_player.mcts == None:\n",
    "#             print('CURRENT PLAYER MCTS: ' + '0' )\n",
    "#         else:\n",
    "#             print('CURRENT PLAYER MCTS: ' + str(len(current_player.mcts)))\n",
    "        \n",
    "        while done == 0:\n",
    "            turn = turn + 1\n",
    "            # env.render()\n",
    "            for r in range(5):\n",
    "                logger_main.info(state.board[5*r : (5*r + 5)])\n",
    "\n",
    "\n",
    "            #### Run the MCTS algo and return an action\n",
    "            if turn < 5:\n",
    "                action, pi = best_player.act(state, 1)\n",
    "            else:\n",
    "                action, pi = best_player.act(state, 0)\n",
    "\n",
    "            logger_main.info('action: %d', action)\n",
    "            logger_main.info('pi: %s', pi)\n",
    "\n",
    "            ####Commit the move to memory\n",
    "            env.remember(state, pi)\n",
    "\n",
    "            #### Do the action\n",
    "            next_state, reward, done, _ = env.step(action)\n",
    "\n",
    "            if done == 1:\n",
    "\n",
    "                #### If the game is finished, get the reward\n",
    "                for move in env.stmemory:\n",
    "                    if move[3] == state.playerTurn:\n",
    "                        move.append(reward)\n",
    "                    else:\n",
    "                        move.append(-reward)\n",
    "                        \n",
    "                logger_memory.info('====================')\n",
    "                logger_memory.info('ADDING NEW MEMORIES')\n",
    "                logger_memory.info('====================')\n",
    "                for s in env.stmemory:\n",
    "                    logger_memory.info('NEW MEMORY: %s', s)\n",
    "\n",
    "                best_player.mcts = None\n",
    "\n",
    "                env.commit_stmemory()\n",
    "\n",
    "\n",
    "\n",
    "            else:\n",
    "                ### Switch to the next state\n",
    "                state = next_state\n",
    "                logger_main.info('board %s', state.board)\n",
    "\n",
    "\n",
    "    #print('retraining...')  \n",
    "    logger_memory.info('====================')\n",
    "    logger_memory.info('CURRENT MEMORY SIZE')\n",
    "    logger_memory.info('====================')\n",
    "    logger_memory.info(len(env.memory))\n",
    "\n",
    "    if len(env.memory) == env.MEMORY_SIZE:\n",
    "        current_player.replay(BATCH_SIZE, env.memory)\n",
    "    \n",
    "        preds_best = best_player.predict(np.array([e[1].convertToModelInput() for e in env.memory]))\n",
    "        preds_current = current_player.predict(np.array([e[1].convertToModelInput() for e in env.memory]))\n",
    "\n",
    "        y_true = tf.constant(np.array([np.append(m[4], m[2]) for m in env.memory]), dtype='float64')\n",
    "        y_pred_best = tf.constant(preds_best, dtype='float64')\n",
    "        y_pred_current = tf.constant(preds_current, dtype='float64')\n",
    "\n",
    "        v_best = tf.slice(y_pred_best, [0,0], [-1,1])\n",
    "        v_current = tf.slice(y_pred_current, [0,0], [-1,1])\n",
    "        z = tf.slice(y_true, [0,0], [-1,1])\n",
    "\n",
    "        #print('TANH VALUE')\n",
    "        #print(sess.run(tf.tanh(v)))\n",
    "        #print('ACTUAL WINNER')\n",
    "        #print(sess.run(z))\n",
    "\n",
    "        p_best = tf.slice(y_pred_best, [0,1], [-1,-1])\n",
    "        p_current = tf.slice(y_pred_current, [0,1], [-1,-1])\n",
    "        pi = tf.slice(y_true, [0,1], [-1,-1])\n",
    "\n",
    "\n",
    "        loss1_best = losses.mean_squared_error(z, tf.tanh(v_best))\n",
    "        loss2_best = tf.nn.softmax_cross_entropy_with_logits(labels = pi, logits = p_best)\n",
    "\n",
    "        total_loss1_best = tf.reduce_mean(loss1_best)\n",
    "        total_loss2_best = tf.reduce_mean(loss2_best)\n",
    "\n",
    "\n",
    "        print('BEST LOSSES ', sess.run(total_loss1_best), ' ', sess.run(total_loss2_best))\n",
    "\n",
    "        loss1_current = losses.mean_squared_error(z, tf.tanh(v_current))\n",
    "        loss2_current = tf.nn.softmax_cross_entropy_with_logits(labels = pi, logits = p_current)\n",
    "\n",
    "        total_loss1_current = tf.reduce_mean(loss1_current)\n",
    "        total_loss2_current = tf.reduce_mean(loss2_current)\n",
    "\n",
    "        print('CURRENT LOSSES ', sess.run(total_loss1_current), ' ', sess.run(total_loss2_current))\n",
    "\n",
    "\n",
    "\n",
    "        #best_score, best_scorecard, best_guesses = best_player.takeExam()\n",
    "        #cp_score, cp_scorecard, cp_guesses = current_player.takeExam()\n",
    "\n",
    "        #print('best_player scores ', best_score, ' out of 10 ', best_scorecard, ' ', best_guesses)\n",
    "        #print('current_player scores ', cp_score, ' out of 10 ', cp_scorecard, ' ', cp_guesses)\n",
    "\n",
    "\n",
    "    #    print('FINISHED RETRAINING')\n",
    "    #     if best_player.mcts == None:\n",
    "    #         print('BEST PLAYER MCTS: ' + '0')\n",
    "    #     else:\n",
    "    #         print('BEST PLAYER MCTS: ' + str(len(best_player.mcts)))\n",
    "\n",
    "    #     if best_player.mcts == None:\n",
    "    #         print('CURRENT PLAYER MCTS: ' + '0' )\n",
    "    #     else:\n",
    "    #         print('CURRENT PLAYER MCTS: ' + str(len(current_player.mcts)))\n",
    "\n",
    "\n",
    "    #     l1 = current_player.model.layers[0].get_weights()[0]\n",
    "    #     plt.imshow(l1, cmap='coolwarm', interpolation='nearest')\n",
    "    #     plt.show()\n",
    "\n",
    "\n",
    "    #     l2 = current_player.model.layers[1].get_weights()[0]\n",
    "    #     plt.imshow(l2, cmap='coolwarm', interpolation='nearest')\n",
    "    #     plt.show()\n",
    "\n",
    "    #     l3 = current_player.model.layers[2].get_weights()[0]\n",
    "    #     plt.imshow(l3, cmap='coolwarm', interpolation='nearest')\n",
    "    #     plt.show()\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "        for e in range(EVAL_EPISODES):\n",
    "\n",
    "\n",
    "            logger_main.info('=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*')\n",
    "            logger_main.info('=*=*=*=*=*=.    TOURNAMENT      =*=*=*=*=*')\n",
    "            logger_main.info('=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*')\n",
    "\n",
    "\n",
    "            state = env.reset()\n",
    "            done = 0\n",
    "            turn = 0\n",
    "    #         print('STARTING TOURNEY MATCH')  \n",
    "    #         if best_player.mcts == None:\n",
    "    #             print('BEST PLAYER MCTS: ' + '0')\n",
    "    #         else:\n",
    "    #             print('BEST PLAYER MCTS: ' + str(len(best_player.mcts)))\n",
    "\n",
    "    #         if best_player.mcts == None:\n",
    "    #             print('CURRENT PLAYER MCTS: ' + '0' )\n",
    "    #         else:\n",
    "    #             print('CURRENT PLAYER MCTS: ' + str(len(current_player.mcts)))\n",
    "\n",
    "            bestPlayerStarts = random.randint(0,1)\n",
    "            if bestPlayerStarts == 1:\n",
    "                players = {1:{\"agent\": best_player, \"name\":\"best_player\"}\n",
    "                        , -1: {\"agent\": current_player, \"name\":\"current_player\"}\n",
    "                        }\n",
    "                logger_main.info('best_player plays as X')\n",
    "            else:\n",
    "                players = {1:{\"agent\": current_player, \"name\":\"current_player\"}\n",
    "                        , -1: {\"agent\": best_player, \"name\":\"best_player\"}\n",
    "                        }\n",
    "                logger_main.info('current_player plays as X')\n",
    "\n",
    "\n",
    "            while done == 0:\n",
    "                turn = turn + 1\n",
    "                for r in range(5):\n",
    "                    logger_main.info([pieces[str(x)] for x in state.board[5*r : (5*r + 5)]])\n",
    "\n",
    "\n",
    "                #### Run the MCTS algo and return an action\n",
    "                if turn < 5:\n",
    "                    action, actionValues = players[state.playerTurn]['agent'].act(state, 1)\n",
    "                else:\n",
    "                    action, actionValues = players[state.playerTurn]['agent'].act(state, 0)\n",
    "\n",
    "                #print('action', action)\n",
    "                #print('pi', actionValues)\n",
    "\n",
    "                #### Do the action\n",
    "                next_state, reward, done, _ = env.step(action)\n",
    "\n",
    "                if done == 1:\n",
    "                    logger_main.info('--------------')\n",
    "                    for r in range(5):\n",
    "                        logger_main.info([pieces[str(x)] for x in next_state.board[5*r : (5*r + 5)]])\n",
    "\n",
    "                    if reward == 1:\n",
    "                        logger_main.info('%s WINS!', players[state.playerTurn]['name'])\n",
    "                        scores[players[state.playerTurn]['name']] = scores[players[state.playerTurn]['name']] + 1\n",
    "                    elif reward == -1:\n",
    "                        logger_main.info('%s WINS!', players[-state.playerTurn]['name'])\n",
    "                        scores[players[-state.playerTurn]['name']] = scores[players[-state.playerTurn]['name']] + 1\n",
    "                    else:\n",
    "                        logger_main.info('DRAW...')\n",
    "                        scores['drawn'] = scores['drawn'] + 1\n",
    "\n",
    "\n",
    "    #                 print('FINISHED TOURNEY MATCH')    \n",
    "    #                 if best_player.mcts == None:\n",
    "    #                     print('BEST PLAYER MCTS: ' + '0')\n",
    "    #                 else:\n",
    "    #                     print('BEST PLAYER MCTS: ' + str(len(best_player.mcts)))\n",
    "\n",
    "    #                 if best_player.mcts == None:\n",
    "    #                     print('CURRENT PLAYER MCTS: ' + '0' )\n",
    "    #                 else:\n",
    "    #                     print('CURRENT PLAYER MCTS: ' + str(len(current_player.mcts)))\n",
    "\n",
    "                    best_player.mcts = None\n",
    "                    current_player.mcts = None\n",
    "\n",
    "                else:\n",
    "                    ### Switch to the next state\n",
    "                    state = next_state\n",
    "                    logger_main.info('--------------')\n",
    "\n",
    "\n",
    "            logger_main.info('=========================')\n",
    "\n",
    "\n",
    "        print(scores)\n",
    "        print()\n",
    "\n",
    "        if scores['current_player'] > scores['best_player'] * 1.3:\n",
    "            best_player_version = best_player_version + 1\n",
    "            best_model = clone_model(current_player.model)\n",
    "            best_model.set_weights(current_player.model.get_weights())\n",
    "            best_player = agent.Agent(state_size, action_size, MCTSsimulations)\n",
    "            best_player.model = best_model\n",
    "            best_players.append(best_player)\n",
    "            env.memory = deque(maxlen=env.MEMORY_SIZE)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.04863173  0.03873249  0.03328731  0.04254941  0.08622798  0.03890872\n",
      "  0.07618684  0.04072398  0.0778295   0.04671157  0.03788808  0.04681649\n",
      "  0.07072482  0.04597522  0.05724363  0.05112813  0.06948191  0.03667006\n",
      "  0.05428204]\n",
      "[ 2  3  5  6  7  9 10 11 13 14 15 16 17 18 19 21 22 23 24]\n",
      "[[7]]\n"
     ]
    }
   ],
   "source": [
    "board = [1 , -1,  0, 0, 1,\n",
    "         0,   0,  0, -1, 0,\n",
    "         0 ,  0 , -1, 0, 0, \n",
    "         0 ,  0 , 0, 0, 0, \n",
    "         1 ,  0 , 0, 0, 0, \n",
    "        ]\n",
    "currentPlayer = 1\n",
    "answer = 20\n",
    "state = game.GameState(np.array(board, dtype=np.int), currentPlayer)\n",
    "inputToModel = np.array([state.convertToModelInput()])\n",
    "preds = current_player.model.predict(inputToModel)[0]\n",
    "\n",
    "logits = preds[1:]\n",
    "odds = np.exp(logits)\n",
    "allowedActions = state.allowedActions()\n",
    "odds = odds[allowedActions]\n",
    "\n",
    "probs = odds / np.sum(odds) ###put this just before the for?\n",
    "\n",
    "\n",
    "guess = np.argwhere(probs == max(probs))\n",
    "\n",
    "\n",
    "print(probs)\n",
    "print(allowedActions)\n",
    "print(allowedActions[guess])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Error when checking : expected conv2d_2_input to have 4 dimensions, but got array with shape (2, 3, 3)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-4-a7594da67250>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mbest_player\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtakeExam\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/Users/davidfoster/Git/AGI/scripts/agent.pyc\u001b[0m in \u001b[0;36mtakeExam\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    306\u001b[0m \t\t]\n\u001b[1;32m    307\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 308\u001b[0;31m                 \u001b[0;32mfor\u001b[0m \u001b[0mq\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mquestions\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    309\u001b[0m                         \u001b[0mq_result\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mq_guess\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtakeQuestion\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mq\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mq\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mq\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    310\u001b[0m                         \u001b[0mscore\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mscore\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mq_result\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/davidfoster/Git/AGI/scripts/agent.pyc\u001b[0m in \u001b[0;36mtakeQuestion\u001b[0;34m(self, board, currentPlayer, answers)\u001b[0m\n\u001b[1;32m    317\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    318\u001b[0m                 \u001b[0mstate\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgame\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mGameState\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mboard\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mint\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcurrentPlayer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 319\u001b[0;31m                 \u001b[0minputToModel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconvertToModelInput\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    320\u001b[0m                 \u001b[0mpreds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputToModel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    321\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/keras/models.pyc\u001b[0m in \u001b[0;36mpredict\u001b[0;34m(self, x, batch_size, verbose)\u001b[0m\n\u001b[1;32m    911\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuilt\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    912\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuild\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 913\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mverbose\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    914\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    915\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mpredict_on_batch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/keras/engine/training.pyc\u001b[0m in \u001b[0;36mpredict\u001b[0;34m(self, x, batch_size, verbose, steps)\u001b[0m\n\u001b[1;32m   1693\u001b[0m         x = _standardize_input_data(x, self._feed_input_names,\n\u001b[1;32m   1694\u001b[0m                                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_feed_input_shapes\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1695\u001b[0;31m                                     check_batch_axis=False)\n\u001b[0m\u001b[1;32m   1696\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstateful\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1697\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0mbatch_size\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mbatch_size\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/keras/engine/training.pyc\u001b[0m in \u001b[0;36m_standardize_input_data\u001b[0;34m(data, names, shapes, check_batch_axis, exception_prefix)\u001b[0m\n\u001b[1;32m    130\u001b[0m                                  \u001b[0;34m' to have '\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mshapes\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    131\u001b[0m                                  \u001b[0;34m' dimensions, but got array with shape '\u001b[0m \u001b[0;34m+\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 132\u001b[0;31m                                  str(array.shape))\n\u001b[0m\u001b[1;32m    133\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mdim\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mref_dim\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshapes\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    134\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mj\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mcheck_batch_axis\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: Error when checking : expected conv2d_2_input to have 4 dimensions, but got array with shape (2, 3, 3)"
     ]
    }
   ],
   "source": [
    "best_player.takeExam()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "124"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(env.memory)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(env.memory)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "tmp = np.array([1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0])\n",
    "for j in range(len(env.memory)):\n",
    "    to_print = 1\n",
    "    for k in range(3): #len(env.memory[j][0])\n",
    "        if env.memory[j][0][k] != tmp[k] or np.sum(np.abs(env.memory[j][0])) != 4:\n",
    "            to_print = 0\n",
    "    if to_print == 1:\n",
    "        print (j)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1 1 0 0 0 0 0 0 0 0 0 1 0 1 0 0 0 0]\n",
      "[1 1 0 0 0 0 0 0 0 0 0 1 0 1 0 0 0 0]\n",
      "[1 1 0 0 0 0 0 0 0 0 0 1 0 1 0 0 0 0]\n"
     ]
    }
   ],
   "source": [
    "print(env.memory[47][0])\n",
    "print(env.memory[159][0])\n",
    "print(env.memory[299][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[array([1, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1]), array([ 0.        ,  0.        ,  0.        ,  0.86486486,  0.        ,\n",
      "        0.13513514,  0.        ,  0.        ,  0.        ]), 1, 1]]\n",
      "[[array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]), array([ 0.        ,  0.        ,  0.        ,  0.        ,  0.15789474,\n",
      "        0.        ,  0.        ,  0.05263158,  0.78947368]), 1, 1]]\n",
      "[[array([0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0]), array([ 0.05882353,  0.02941176,  0.05882353,  0.02941176,  0.64705882,\n",
      "        0.02941176,  0.08823529,  0.05882353,  0.        ]), -1, -1]]\n",
      "[[array([0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0]), array([ 0.48780488,  0.04878049,  0.14634146,  0.07317073,  0.        ,\n",
      "        0.07317073,  0.12195122,  0.04878049,  0.        ]), 1, 1]]\n",
      "[[array([1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0]), array([ 0.        ,  0.05714286,  0.57142857,  0.08571429,  0.        ,\n",
      "        0.05714286,  0.14285714,  0.08571429,  0.        ]), -1, -1]]\n",
      "[[array([1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0]), array([ 0.   ,  0.075,  0.   ,  0.025,  0.   ,  0.075,  0.75 ,  0.075,  0.   ]), 1, 1]]\n",
      "[[array([1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0]), array([ 0.        ,  0.34042553,  0.        ,  0.10638298,  0.        ,\n",
      "        0.14893617,  0.        ,  0.40425532,  0.        ]), -1, -1]]\n",
      "[[array([1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 0, 1, 0, 0, 1, 0]), array([ 0.        ,  0.02631579,  0.        ,  0.92105263,  0.        ,\n",
      "        0.05263158,  0.        ,  0.        ,  0.        ]), 1, 1]]\n"
     ]
    }
   ],
   "source": [
    "idx = 80\n",
    "for j in range(8):\n",
    "    print(list(itertools.islice(env.memory, idx + 4 * j, idx + 4 * j + 1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': [1.7576870526840438, 1.6732365248808221, 1.6130136393789034, 1.5743154554224725, 1.5411098999763602], 'val_loss': [1.7178926608141731, 1.6698948565651388, 1.6472153102650362, 1.6012159024967867, 1.5635613904279821]}\n",
      "{'loss': [1.5107648283687991, 1.4877198418574547, 1.4616420962917271, 1.4480195383527386, 1.4325322250821697], 'val_loss': [1.6092261496712179, 1.5876388199189131, 1.6036715437384212, 1.6026649334851433, 1.6104948731029736]}\n",
      "{'loss': [1.4440990134851257, 1.4299852901430272, 1.4140669125229566, 1.416490174051541, 1.4035840532672939], 'val_loss': [1.5447220241322237, 1.5371169973822201, 1.5466907445122213, 1.5338563358082491, 1.5169404254240149]}\n",
      "{'loss': [1.4187768003833827, 1.4077386945041257, 1.4096526245572674, 1.3996178071890304, 1.3913287137871357], 'val_loss': [1.4184219977434944, 1.4272943875368904, 1.4321957756491268, 1.4231686451855827, 1.4186213086633122]}\n",
      "{'loss': [1.3935285966787765, 1.3845354656674969, 1.3908711458320049, 1.3849976810056772, 1.3711245736079429], 'val_loss': [1.4345435114467846, 1.4509663511725033, 1.4808147093828987, 1.4578366209478939, 1.4496014188317692]}\n",
      "{'loss': [1.4025918014013945, 1.3989239877729274, 1.3945754624124784, 1.3900402001480558, 1.3894166928618701], 'val_loss': [1.321506935007432, 1.3177969315472771, 1.3410503794165218, 1.321259617805481, 1.3269541614195879]}\n",
      "{'loss': [1.3879896171057402, 1.3778269148584623, 1.3699338738598041, 1.371442520796363, 1.3772981576065519], 'val_loss': [1.3217957370421465, 1.3322044961592729, 1.3444842941620772, 1.34552404459785, 1.3531705772175509]}\n",
      "{'loss': [1.3812796108758272, 1.3701794200868749, 1.3642830119204166, 1.3596654632198277, 1.3539389069400616], 'val_loss': [1.3448070778566248, 1.348842459566453, 1.3523389030905331, 1.3493271364885218, 1.3549591302871704]}\n",
      "{'loss': [1.3658378711387293, 1.3569889976017511, 1.3540920161489229, 1.3506073489117978, 1.3601186844839979], 'val_loss': [1.3281341931399178, 1.3409276850083296, 1.3393094609765446, 1.3454897193347706, 1.3443664312362671]}\n",
      "{'loss': [1.3325481219078177, 1.3277667814226293, 1.3293116323983492, 1.3240687491288825, 1.3202969636490096], 'val_loss': [1.4171383941874784, 1.4219791819067562, 1.4288714493022245, 1.4317997764138615, 1.427109066177817]}\n",
      "{'loss': [1.3762733971894676, 1.3709996049083881, 1.3767525260128193, 1.3705549684923086, 1.3647839542645126], 'val_loss': [1.2111879552111906, 1.2194731200442595, 1.2299447340123795, 1.2141031622886658, 1.2197803925065434]}\n",
      "{'loss': [1.3176870488408785, 1.3122383569603535, 1.3095057864687336, 1.3126976276511577, 1.3114769209676713], 'val_loss': [1.4133882873198564, 1.4210352827520931, 1.4322272118400126, 1.4249251730301802, 1.4294756791170906]}\n",
      "{'loss': [1.342134742594477, 1.3374191184542072, 1.3366641411140783, 1.3393635518515288, 1.3348568393223321], 'val_loss': [1.3142689887215109, 1.326888638384202, 1.3220881924909704, 1.3289619754342472, 1.3210580208722282]}\n",
      "{'loss': [1.3571070752926726, 1.3557942571924693, 1.3642027983024938, 1.3547579050064087, 1.3446945201105147], 'val_loss': [1.2496164335924036, 1.2615663794910206, 1.2642765325658463, 1.2633644552791821, 1.2539629164864035]}\n",
      "{'loss': [1.3144988266389761, 1.3201285778586545, 1.3176390555367541, 1.3207753612034356, 1.3131239556554537], 'val_loss': [1.3814472030190861, 1.3934464454650879, 1.4116572772755343, 1.3755075440687292, 1.3984673724454992]}\n",
      "{'loss': [1.3212322530461782, 1.3133410055245927, 1.3198929544705063, 1.3200539332717212, 1.3110441930258452], 'val_loss': [1.3656038256252514, 1.4017717908410465, 1.410092508091646, 1.3851037937052109, 1.3903228465248556]}\n",
      "{'loss': [1.3220234771273029, 1.3100228220669192, 1.305954223248496, 1.310403795384649, 1.3077644483367008], 'val_loss': [1.3674085631090052, 1.37529377376332, 1.3943573657204122, 1.3953802375232471, 1.4084265512578629]}\n",
      "{'loss': [1.3408583313671512, 1.3328248671631315, 1.3296257061744803, 1.3260856724497099, 1.328683824681524], 'val_loss': [1.2818075278226067, 1.2796143363503849, 1.286549771533293, 1.2887775687610401, 1.2953214294770186]}\n",
      "{'loss': [1.3262071040139269, 1.3237869561608158, 1.3264150939770598, 1.3210541330166716, 1.3178283011735374], 'val_loss': [1.3117407069486731, 1.3314641643972958, 1.330417794339797, 1.3224636105930103, 1.3261892865685856]}\n",
      "{'loss': [1.3212044648270109, 1.3169022318142563, 1.3114540612519676, 1.3144040748254577, 1.3251054963069175], 'val_loss': [1.3398451384376078, 1.3214381652719833, 1.3283938449971817, 1.3436738252639771, 1.3339586959165686]}\n",
      "{'loss': [1.3050542037878463, 1.3190798545951274, 1.3136524887227301, 1.3048927748381203, 1.3014741584436218], 'val_loss': [1.3772095371695126, 1.3858258303473978, 1.4051955447477453, 1.4001594361136942, 1.3923107245389152]}\n",
      "{'loss': [1.3179137226360946, 1.3136016824352208, 1.3139686566680224, 1.3145736349162771, 1.3123142737061231], 'val_loss': [1.301545262336731, 1.314270861008588, 1.3058169098461376, 1.3137499374501846, 1.307552176363328]}\n",
      "{'loss': [1.3017919010190822, 1.2989954912840431, 1.3013080305127955, 1.3038904133127696, 1.2993911220066583], 'val_loss': [1.3483870099572575, 1.3560375816681807, 1.3556087367674883, 1.3529195434906904, 1.363927893778857]}\n",
      "{'loss': [1.2991152983992846, 1.2964181935609276, 1.2984224059688512, 1.2933046568685502, 1.2945784348160474], 'val_loss': [1.3763097103904276, 1.3845760611926807, 1.3816072800580192, 1.3835036894854378, 1.3786554196301628]}\n",
      "{'loss': [1.2895283699035645, 1.2933587547558456, 1.2872803371344039, 1.2868062515756977, 1.2838113948480407], 'val_loss': [1.4029672776951509, 1.3974561060176176, 1.3984546240638285, 1.4093012809753418, 1.4056659866781795]}\n",
      "{'loss': [1.3192810834343753, 1.3091305654440353, 1.3116964208545969, 1.3101966060809236, 1.3092903094505197], 'val_loss': [1.2954561079249662, 1.3141292614095352, 1.3207520526998184, 1.3188254692975212, 1.3164144193424898]}\n",
      "{'loss': [1.2955561894089429, 1.2950074352435212, 1.2924749388623593, 1.2981278220219399, 1.2906451082941313], 'val_loss': [1.3730041770374073, 1.3763360381126404, 1.3870475993436926, 1.3782105936723597, 1.3893382970024557]}\n",
      "{'loss': [1.3015779541499579, 1.2965764127560515, 1.2943856075628479, 1.3019043135998853, 1.2934083120146793], 'val_loss': [1.342876343166127, 1.3540541143978344, 1.3499938389834236, 1.3502775921541101, 1.3723982642678654]}\n",
      "{'loss': [1.3361812207236219, 1.3322859112896137, 1.3359008244614103, 1.3324786513598996, 1.331858654520405], 'val_loss': [1.2216366178849165, 1.2158466787899243, 1.232502867193783, 1.2199953864602482, 1.2173197760301477]}\n",
      "{'loss': [1.291120296093955, 1.2908579271231124, 1.2875354983913365, 1.281503760992591, 1.2817377492563049], 'val_loss': [1.3818159033270443, 1.4007643320981193, 1.3911789375192978, 1.3960357694064869, 1.3989493005415972]}\n",
      "{'loss': [1.2974569406082381, 1.2982921208908309, 1.2960283204690735, 1.2940281576185084, 1.2990212831924211], 'val_loss': [1.3422213442185347, 1.35140079610488, 1.3556673877379473, 1.3596232007531559, 1.3611934044781853]}\n",
      "{'loss': [1.3189137818208381, 1.3176234747046855, 1.3299315900944952, 1.3131319391193674, 1.3164748451602992], 'val_loss': [1.3158821218154009, 1.3303959580028759, 1.3327683560988481, 1.3495789696188534, 1.3307340145111084]}\n",
      "{'loss': [1.295976551611032, 1.2899424816245464, 1.2907397569115482, 1.287972567686394, 1.2864626539287283], 'val_loss': [1.3694868578630335, 1.3895774308372946, 1.3773853989208447, 1.3829385112313664, 1.3822657220503862]}\n",
      "{'loss': [1.30083637451058, 1.298791694996962, 1.3009583967835157, 1.2972659228452996, 1.2995773269169366], 'val_loss': [1.3401536661035873, 1.3503378910176895, 1.3410551477881039, 1.3602901907528149, 1.3353150522007662]}\n",
      "{'loss': [1.2873944222037472, 1.286703641734906, 1.299390143422938, 1.2956250318840368, 1.2970743250491015], 'val_loss': [1.3824172510820276, 1.4138591009027817, 1.4040156532736385, 1.4070583581924438, 1.3861706677605123]}\n",
      "{'loss': [1.3221936670701895, 1.3080230453121129, 1.3064531248007247, 1.3047230350437449, 1.3011446123692527], 'val_loss': [1.332835639224333, 1.348252065041486, 1.3307609487982357, 1.3341438980663525, 1.329020717564751]}\n",
      "{'loss': [1.2990043679280068, 1.2984971946744777, 1.3024460041700905, 1.2930664429024084, 1.2975977427923857], 'val_loss': [1.3376676335054285, 1.36186024721931, 1.3643574434168197, 1.3497519142487471, 1.3574633878820084]}\n",
      "{'loss': [1.2782591278873272, 1.2767564271813008, 1.2679341159649749, 1.2654213122467497, 1.2646421976943514], 'val_loss': [1.4135809716056376, 1.4561690442702349, 1.4485694801106173, 1.4605563949136173, 1.4631770358366125]}\n",
      "{'loss': [1.2858437886878626, 1.2798328755506829, 1.2808258586855077, 1.2796075593179732, 1.2817846678975802], 'val_loss': [1.3740443762610941, 1.3733882413190954, 1.3933407909729902, 1.3783951296525843, 1.3824036612230188]}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': [1.3153550126659337, 1.3145343385525603, 1.3163579595622732, 1.3128921843286772, 1.3134496674608829], 'val_loss': [1.2772673578823315, 1.2778545618057251, 1.2812263194252462, 1.2883936938117533, 1.2890534190570606]}\n",
      "{'loss': [1.298452393332524, 1.2949666763419536, 1.2973461293462496, 1.2914675420789576, 1.3010728572731587], 'val_loss': [1.3567719459533691, 1.3681871119667501, 1.3887485335854923, 1.3823070455999935, 1.380059284322402]}\n",
      "{'loss': [1.3173990356388376, 1.3107635690205133, 1.3098528011521298, 1.317295220360827, 1.3081252877391987], 'val_loss': [1.2724496336544262, 1.2737195842406328, 1.2784034574733061, 1.2863775281345142, 1.26426582476672]}\n",
      "{'loss': [1.2712517852213845, 1.2734171668095375, 1.2727093999065571, 1.2710269653975075, 1.2684040265296823], 'val_loss': [1.4035313199548161, 1.427655023687026, 1.4213481440263636, 1.428042033139397, 1.4388583057066973]}\n",
      "{'loss': [1.2814140764635, 1.2695385431175801, 1.2759355466757247, 1.2779471678520316, 1.2838272123194452], 'val_loss': [1.4131255780949312, 1.41593617551467, 1.4185584923800301, 1.4549767900915707, 1.4375455449609196]}\n",
      "{'loss': [1.3309714473895173, 1.3216489197602912, 1.3106909011726948, 1.3070557597857804, 1.3073191020026136], 'val_loss': [1.2664730268366196, 1.2729213728624231, 1.2689211859422571, 1.2833331122117884, 1.2682136016733505]}\n",
      "{'loss': [1.2923674174209139, 1.2924713216610808, 1.2868169724051632, 1.2869169961160689, 1.286454481865043], 'val_loss': [1.3265980061362772, 1.3339898095411413, 1.3407349866979263, 1.3413082992329317, 1.339699015897863]}\n",
      "{'loss': [1.2886790837814559, 1.2897055558304289, 1.2860149305258224, 1.2884384233560136, 1.2850138945366019], 'val_loss': [1.3221198741127462, 1.3340320727404427, 1.342239127439611, 1.3443702108719771, 1.3391216572593241]}\n",
      "{'loss': [1.2866276805080585, 1.2935577435279959, 1.289853894888465, 1.2844061744746877, 1.2859119675052699], 'val_loss': [1.3249205000260298, 1.3438236783532536, 1.3429141044616699, 1.3364662563099581, 1.3420269068549662]}\n",
      "{'loss': [1.2858802080154419, 1.2858099670552496, 1.2924107188609109, 1.296116647435658, 1.2841357917927985], 'val_loss': [1.3336126734228695, 1.338480128961451, 1.3602257055394791, 1.3422960533815271, 1.3443748670465805]}\n",
      "{'loss': [1.2909540258236785, 1.2892264394617792, 1.2908117895695701, 1.2911190559614951, 1.291010995409382], 'val_loss': [1.3322719475802254, 1.3498834511812996, 1.3419382782543408, 1.3483743807848763, 1.3602581865647261]}\n",
      "{'loss': [1.3094605047311356, 1.3054434399106609, 1.3030626435778034, 1.3000161558834475, 1.3016649715936006], 'val_loss': [1.281426001997555, 1.2794206843656653, 1.2806363526512594, 1.2786915091907276, 1.2778013313517851]}\n",
      "{'loss': [1.3019566340232962, 1.3003553757027013, 1.2985031925030608, 1.3020275101732852, 1.2990721926760318], 'val_loss': [1.2693278509027817, 1.2729384057662065, 1.2821043379166548, 1.281917557996862, 1.2859456118415384]}\n",
      "{'loss': [1.2934335238897978, 1.2912154589126359, 1.2984830532500993, 1.2923301849792252, 1.2853018579198354], 'val_loss': [1.3158896249883316, 1.3266465804156136, 1.339261580916012, 1.3449371842777027, 1.3414085191838883]}\n",
      "{'loss': [1.3092320321211175, 1.3063929525773916, 1.3110203778565819, 1.3100702531302153, 1.3152545299103011], 'val_loss': [1.2517176866531372, 1.2558679931304033, 1.2647203136892879, 1.2714503232170553, 1.2735925071379717]}\n",
      "{'loss': [1.3197080039266329, 1.3194287054574312, 1.315625213865024, 1.3115994521041414, 1.3142492219583313], 'val_loss': [1.2552782998365515, 1.2584356560426599, 1.2561144127565271, 1.2651799636728622, 1.262002103468951]}\n",
      "{'loss': [1.3093216721691303, 1.3091789697533223, 1.3145226115611062, 1.3166980921332516, 1.3104046084987584], 'val_loss': [1.2517605529111975, 1.2667623688192928, 1.2585676908493042, 1.292794339797076, 1.2558771722456987]}\n",
      "{'loss': [1.2987758775255573, 1.3088663852036888, 1.3008563251637701, 1.2950759752472836, 1.2980782060480829], 'val_loss': [1.314861486939823, 1.312785169657539, 1.3080420143464033, 1.3100075020509607, 1.3140958126853495]}\n",
      "{'loss': [1.2899225078411956, 1.278785242963193, 1.2826369057840377, 1.2805408168194898, 1.282280977092572], 'val_loss': [1.3673919018577128, 1.3744889427633846, 1.3910234114703011, 1.3941907111336203, 1.3820070659413057]}\n",
      "{'loss': [1.2944850957215721, 1.2955629380781259, 1.2925827182940584, 1.2949149661989354, 1.3041636214327457], 'val_loss': [1.3056994957082413, 1.3039399385452271, 1.3240297962637508, 1.3049508263083065, 1.3338593945783728]}\n",
      "{'loss': [1.2722594044101772, 1.2665942320183141, 1.2626491215691638, 1.2639809686746171, 1.2622137550097794], 'val_loss': [1.414228635675767, 1.4093977633644552, 1.4185867730308981, 1.4103522861705107, 1.4112243792589974]}\n",
      "{'loss': [1.2827814728466433, 1.283406389293386, 1.2827611805787726, 1.2864995554311951, 1.2899724102731962], 'val_loss': [1.3253520516788257, 1.3410231155507706, 1.3372876644134521, 1.3369783022824455, 1.3468021154403687]}\n",
      "{'loss': [1.2881916547889141, 1.2834764256406186, 1.28341549368047, 1.2825753564265236, 1.2821995553685659], 'val_loss': [1.3337283695445341, 1.3390188427532421, 1.3424345956129187, 1.3500049885581522, 1.343218207359314]}\n",
      "{'loss': [1.3094880758826413, 1.3085790000744719, 1.3077898666040222, 1.3048265585258825, 1.3118387264991873], 'val_loss': [1.2288180168937235, 1.2258834418128519, 1.2289675684536205, 1.2391998908099007, 1.2365239508011763]}\n",
      "{'loss': [1.3048827274521786, 1.3001930517936819, 1.2973566873749691, 1.3075816168713925, 1.2977732882570865], 'val_loss': [1.2892394346349381, 1.2917094651390524, 1.2916355273302864, 1.3026864598779118, 1.3078081748064827]}\n",
      "{'loss': [1.3020384845448965, 1.2850005057320666, 1.283634819201569, 1.283647601284198, 1.2837834020159138], 'val_loss': [1.3495834995718563, 1.3176344773348641, 1.3479888439178467, 1.3340677864411299, 1.3461268999997307]}\n",
      "{'loss': [1.3086228904439443, 1.3163384554991082, 1.3193806943608755, 1.3090837553365906, 1.3041333155845529], 'val_loss': [1.2381517536499922, 1.2536046329666586, 1.2451232461368336, 1.2468436640851639, 1.2494326128679163]}\n",
      "{'loss': [1.2807206342469399, 1.2757879993808803, 1.2759355395587522, 1.275727337865687, 1.2715459136820551], 'val_loss': [1.3684269330080818, 1.3742030648624195, 1.3893774677725399, 1.3785886834649479, 1.4000500510720646]}\n",
      "{'loss': [1.2980544584900586, 1.2911669353940594, 1.2903702543742621, 1.2918364859339018, 1.293503889397009], 'val_loss': [1.3097928271574133, 1.3054190102745504, 1.3035322638118969, 1.3065327055314009, 1.2978013122782988]}\n",
      "{'loss': [1.2885465871042281, 1.2876364046068334, 1.290746445086465, 1.2838649998849898, 1.2857412544649038], 'val_loss': [1.3274389084647684, 1.3227492851369522, 1.3302044517853682, 1.3196956129635082, 1.3338160164215986]}\n",
      "{'loss': [1.2887189939840515, 1.2865925635864486, 1.2860815418300344, 1.2845736350586165, 1.285549815021344], 'val_loss': [1.3352240043527939, 1.3456546348683975, 1.3595986857133753, 1.3620384370579439, 1.3657494643155266]}\n",
      "{'loss': [1.2888091944936495, 1.2830082338247726, 1.2839163114775474, 1.2836204624887724, 1.2845240706828103], 'val_loss': [1.3327148030785954, 1.3472294036079855, 1.36153165733113, 1.3504984519060921, 1.3628325322095085]}\n",
      "{'loss': [1.3124763912229396, 1.3187666579858581, 1.3080102294238645, 1.3026872090439299, 1.2998001771186716], 'val_loss': [1.2400011876050163, 1.2606658374561983, 1.2472941594965317, 1.2480395751840927, 1.2592558229670805]}\n",
      "{'loss': [1.293416307933295, 1.2863616925566943, 1.2830801543904775, 1.2847253582370814, 1.2844100336530315], 'val_loss': [1.3016100701163797, 1.3035918193704941, 1.311626672744751, 1.3137457160388721, 1.3200221552568323]}\n",
      "{'loss': [1.3028534401708574, 1.3016228996106047, 1.3053029224054138, 1.3028506912402253, 1.3076883643420774], 'val_loss': [1.2374840974807739, 1.2454448517631083, 1.2576339244842529, 1.2642244100570679, 1.2473458752912634]}\n",
      "{'loss': [1.2843346506802005, 1.2839847304927769, 1.2866919129642087, 1.2847337580438871, 1.282616976481765], 'val_loss': [1.3171326553120333, 1.3278613931992476, 1.3209878416622387, 1.34052092888776, 1.3321465183706844]}\n",
      "{'loss': [1.3021355173481044, 1.2987491675277254, 1.2986554352205191, 1.2961127491139655, 1.2934586272310855], 'val_loss': [1.2752264317344217, 1.272879256921656, 1.283389785710503, 1.2698808347477633, 1.2850917717989754]}\n",
      "{'loss': [1.3015418195012789, 1.2995389824482932, 1.3049674105288378, 1.2992998041323762, 1.2971436995178907], 'val_loss': [1.2848681702333338, 1.3064610677606918, 1.3236741739160873, 1.3001266227048986, 1.3018648554297054]}\n",
      "{'loss': [1.3024662181512634, 1.2941952908217018, 1.2951333522796631, 1.2945982164411403, 1.2899672148832635], 'val_loss': [1.2731261393603157, 1.2863685944501091, 1.2884397085975199, 1.291514564962948, 1.2997927876079785]}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': [1.3125592025358286, 1.3140865279667413, 1.3149796083791931, 1.3105250774924435, 1.3108997683026897], 'val_loss': [1.2154587857863481, 1.2223098383230322, 1.2189865673289579, 1.2303095424876493, 1.2245636897928573]}\n",
      "{'loss': [1.2742454578627402, 1.2740195491420689, 1.2672697679320377, 1.2784523928343361, 1.2765892761856763], 'val_loss': [1.3573363388285917, 1.361625916817609, 1.3648469167597153, 1.3839606046676636, 1.3584324962952559]}\n",
      "{'loss': [1.31413855125655, 1.3115447208062927, 1.3105763214737622, 1.3084828444381258, 1.303916585979177], 'val_loss': [1.2399559722227209, 1.2447993965709911, 1.2608657163732193, 1.2464551855536068, 1.2473975419998169]}\n",
      "{'loss': [1.2776199045465952, 1.2753966584134457, 1.2728363969432774, 1.2712496768182784, 1.2724138640645724], 'val_loss': [1.3639616826001335, 1.379619738634895, 1.3776562143774593, 1.3711683960521923, 1.3809087907566744]}\n",
      "{'loss': [1.2802659522241622, 1.277902322029, 1.2767823756630741, 1.2760677355438916, 1.2761582901228721], 'val_loss': [1.3235324761446785, 1.3337193131446838, 1.338211245396558, 1.3406006097793579, 1.3450063992949093]}\n",
      "{'loss': [1.2663032439217639, 1.2596017193438402, 1.2574712411681217, 1.2632094265809699, 1.2587357243495201], 'val_loss': [1.4043220001108505, 1.4252821347292732, 1.4188222534516279, 1.4409370282117058, 1.4155262638540829]}\n",
      "{'loss': [1.3141116259702996, 1.310707729254196, 1.3095175099017016, 1.3087652601412874, 1.3098069198095976], 'val_loss': [1.1890743550132303, 1.2023180582944084, 1.2062109428293564, 1.2075963931925155, 1.2131836133844711]}\n",
      "{'loss': [1.2754007915952312, 1.2738416497387104, 1.283299054672469, 1.2811097155756026, 1.2803832968669151], 'val_loss': [1.3526276840883142, 1.3517010492437027, 1.355702603564543, 1.3467395165387321, 1.3777940273284912]}\n",
      "{'loss': [1.2939445385292394, 1.2891808427981477, 1.2864555956712409, 1.2806921966040312, 1.2800787171321129], 'val_loss': [1.3086127253139721, 1.308409473475288, 1.3132266928167904, 1.3176578633925493, 1.3207805928061991]}\n",
      "{'loss': [1.2827664738270774, 1.2873437226708255, 1.2828083340801411, 1.2809768488158042, 1.2809560494636423], 'val_loss': [1.3166725635528564, 1.3195258098490097, 1.3083898880902458, 1.3108142684487736, 1.3126145531149471]}\n",
      "{'loss': [1.2973010575593407, 1.2988164389311379, 1.3012311351833059, 1.3085494984441728, 1.3032047036868424], 'val_loss': [1.2493786110597498, 1.2537004877539242, 1.2754761401344747, 1.2764219256008373, 1.2578815712648279]}\n",
      "{'loss': [1.3017182403535985, 1.2994058078794337, 1.3053725566436996, 1.3017838374892277, 1.3020007556943751], 'val_loss': [1.2470632651272942, 1.2485454643473906, 1.260999321937561, 1.2640737786012537, 1.2565152995726641]}\n",
      "{'loss': [1.2987032381456289, 1.3017380664597695, 1.2986167010976308, 1.2997358290117178, 1.2932909033191737], 'val_loss': [1.2643086138893576, 1.2746184363084681, 1.2818346374175127, 1.2773287226172054, 1.2704637611613554]}\n",
      "{'loss': [1.3020969985136346, 1.2958167862536303, 1.3024962339828263, 1.2978784361881996, 1.2939522497689546], 'val_loss': [1.269357814508326, 1.2871337988797356, 1.2759507754269768, 1.2841867769465727, 1.2786472474827486]}\n",
      "{'loss': [1.3311385378908756, 1.3341092636336143, 1.3270961473237222, 1.3291941400784164, 1.3252694090800499], 'val_loss': [1.1437273726743811, 1.1555276828653671, 1.1725362539291382, 1.1678440991569967, 1.1809852824491613]}\n",
      "{'loss': [1.269482594817432, 1.270260396288402, 1.263394024834704, 1.2684045033668405, 1.2679882601125916], 'val_loss': [1.411621191922356, 1.4057120014639461, 1.4185403795803295, 1.4126068143283619, 1.4152168175753426]}\n",
      "{'loss': [1.2859506393546489, 1.2834372929672697, 1.2871397626933767, 1.2811145995979878, 1.2833634543774732], 'val_loss': [1.3134055277880501, 1.3349733843522913, 1.3347650556003345, 1.3245974989498364, 1.3194958883173324]}\n",
      "{'loss': [1.3031211095069772, 1.3033584854496059, 1.3025005184002776, 1.3068439177612761, 1.2998416281458158], 'val_loss': [1.2206866039949305, 1.2232783121221207, 1.2347816158743465, 1.2300349684322582, 1.2326292991638184]}\n",
      "{'loss': [1.2756481010522416, 1.2779057719814244, 1.2735524248720995, 1.2720826704110673, 1.271323269872523], 'val_loss': [1.3469206059680265, 1.3410778641700745, 1.3470867872238159, 1.3484036641962387, 1.3459719033802258]}\n",
      "{'loss': [1.2883358588859217, 1.2892476747285073, 1.2842061323906058, 1.2835277443501487, 1.2841703678244976], 'val_loss': [1.2925950358895695, 1.296456449172076, 1.3019810073515947, 1.2963014209971708, 1.2960540757459753]}\n",
      "{'loss': [1.2908771020262988, 1.286500571379021, 1.285569539710657, 1.2887129730253077, 1.2835188250043499], 'val_loss': [1.300231463768903, 1.3078269678003647, 1.2989965817507576, 1.2967170547036564, 1.3137277715346392]}\n",
      "{'loss': [1.3019508621585902, 1.3009753636459807, 1.3007716509833265, 1.2994526055321765, 1.2961157133330161], 'val_loss': [1.2461307609782499, 1.2500081693424898, 1.2548486695570105, 1.2517535616369808, 1.2493592851302202]}\n"
     ]
    }
   ],
   "source": [
    "for i in range(100):\n",
    "    current_player.replay(len(env.memory),env.memory)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0  0 -1  0  0  0  1 -1  1]\n",
      "ACTIONS\n",
      "NEW\n",
      "(0, array([ 0.78947368,  0.05263158,  0.        ,  0.05263158,  0.05263158,\n",
      "        0.05263158,  0.        ,  0.        ,  0.        ]))\n",
      "CURRENT\n",
      "(4, array([ 0.,  0.,  0.,  0.,  1.,  0.,  0.,  0.,  0.]))\n",
      "BEST\n",
      "(4, array([ 0.,  0.,  0.,  0.,  1.,  0.,  0.,  0.,  0.]))\n",
      "NEURAL NETWORKS\n",
      "NEW\n",
      "[-0.61689717  0.04314395 -0.21813801  0.00464708 -0.22530153 -0.23609386\n",
      "  0.53858405 -0.12439886 -0.19880626]\n",
      "[ 0.06426547  0.12434545  0.09575392  0.11964951  0.09507044  0.09404992\n",
      "  0.20407829  0.10516397  0.09762303]\n",
      "CURRENT\n",
      "[ 1.22523439  1.87193167 -2.47061253  1.17246187  2.42772818 -0.52498233\n",
      " -1.03778112 -1.82270682 -1.20306849]\n",
      "[ 0.13115667  0.25040758  0.00325615  0.12441467  0.43654278  0.02278667\n",
      "  0.01364505  0.00622424  0.01156623]\n",
      "BEST\n",
      "[ 1.52912474  0.15564844 -3.02101469  0.64040744  3.38259149  0.51579404\n",
      " -0.80624282 -1.89907336 -1.20535231]\n",
      "[ 0.11608961  0.02939681  0.00122657  0.04773405  0.74087292  0.04214144\n",
      "  0.01123456  0.00376657  0.00753746]\n"
     ]
    }
   ],
   "source": [
    "idx = 80\n",
    "\n",
    "import game\n",
    "\n",
    "#MCTSsimulations = 100\n",
    "#current_player = agent.Agent(state_size, action_size, MCTSsimulations)\n",
    "#best_player = agent.Agent(state_size, action_size, MCTSsimulations)\n",
    "\n",
    "new_player = agent.Agent(state_size, action_size, MCTSsimulations)\n",
    "\n",
    "board = np.array([0,0,-1,0,0,0,1,-1,1])\n",
    "print(board)\n",
    "state = game.GameState(board, 1)\n",
    "\n",
    "new_player.mcts = None\n",
    "current_player.mcts = None\n",
    "best_player.mcts = None\n",
    "\n",
    "print('ACTIONS')\n",
    "print('NEW')\n",
    "print(new_player.act(state, 0))\n",
    "print('CURRENT')\n",
    "print(current_player.act(state, 0))\n",
    "print('BEST')\n",
    "print(best_player.act(state, 0))\n",
    "\n",
    "data = [[np.array(state.binary())]]\n",
    "sess = tf.Session()\n",
    "print('NEURAL NETWORKS')\n",
    "print('NEW')\n",
    "print(new_player.predict(np.array([e[0] for e in data]))[0][1:])\n",
    "print(sess.run(tf.nn.softmax(np.array(new_player.predict(np.array([e[0] for e in data]))[0][1:]))))\n",
    "print('CURRENT')\n",
    "print(current_player.predict(np.array([e[0] for e in data]))[0][1:])\n",
    "print(sess.run(tf.nn.softmax(np.array(current_player.predict(np.array([e[0] for e in data]))[0][1:]))))\n",
    "\n",
    "print('BEST')\n",
    "print(best_player.predict(np.array([e[0] for e in data]))[0][1:])\n",
    "print(sess.run(tf.nn.softmax(np.array(best_player.predict(np.array([e[0] for e in data]))[0][1:]))))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2, array([ 0.,  0.,  1.,  0.,  0.,  0.,  0.,  0.,  0.]))\n",
      "{'Q': 0.98623503595590589, 'P': 1, 'W': 39.449401438236237, 'N': 40}\n",
      "[array([1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0])]\n"
     ]
    }
   ],
   "source": [
    "\n",
    "print(current_player.act(state, 0))\n",
    "print(current_player.mcts.tree[state.convertStateToId()].stats)\n",
    "\n",
    "#print(best_player.act(state, 0))\n",
    "\n",
    "print(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "22"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(best_player.mcts)\n",
    "len(current_player.mcts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LOSSES\n",
      "3.13525017795\n",
      "0.870526376158\n",
      "2.26472380179\n",
      "------\n",
      "0.875415976207\n",
      "PI\n",
      "[[ 0.1010101   0.12121212  0.1010101  ...,  0.1010101   0.1010101\n",
      "   0.13131313]\n",
      " [ 0.1010101   0.11111111  0.1010101  ...,  0.13131313  0.11111111\n",
      "   0.1010101 ]\n",
      " [ 0.13131313  0.1010101   0.1010101  ...,  0.1010101   0.12121212\n",
      "   0.1010101 ]\n",
      " ..., \n",
      " [ 0.          0.          0.96045198 ...,  0.02259887  0.01694915  0.        ]\n",
      " [ 0.02259887  0.          0.         ...,  0.          0.          0.96045198]\n",
      " [ 0.          0.01694915  0.02259887 ...,  0.96045198  0.          0.        ]]\n",
      "PROBS\n",
      "[[ 0.1083045   0.11030342  0.10945559 ...,  0.11207842  0.11971595\n",
      "   0.11085337]\n",
      " [ 0.1083045   0.11030342  0.10945559 ...,  0.11207842  0.11971595\n",
      "   0.11085337]\n",
      " [ 0.1083045   0.11030342  0.10945559 ...,  0.11207842  0.11971595\n",
      "   0.11085337]\n",
      " ..., \n",
      " [ 0.08188468  0.12271193  0.09904243 ...,  0.09753785  0.12103736\n",
      "   0.10803305]\n",
      " [ 0.0427911   0.07212256  0.11233814 ...,  0.12501495  0.25944976\n",
      "   0.039162  ]\n",
      " [ 0.06784247  0.12871168  0.10488583 ...,  0.13192028  0.1672409\n",
      "   0.06691065]]\n",
      "DIFF\n",
      "[[  7.29439741e-03   1.09087001e-02   8.44549091e-03 ...,   1.10683149e-02\n",
      "    1.87058517e-02   2.04597568e-02]\n",
      " [  7.29439741e-03   8.07690046e-04   8.44549091e-03 ...,   1.92347154e-02\n",
      "    8.60484157e-03   9.84327348e-03]\n",
      " [  2.30086329e-02   9.29332006e-03   8.44549091e-03 ...,   1.10683149e-02\n",
      "    1.49616853e-03   9.84327348e-03]\n",
      " ..., \n",
      " [  8.18846829e-02   1.22711929e-01   8.61409545e-01 ...,   7.49389847e-02\n",
      "    1.04088211e-01   1.08033050e-01]\n",
      " [  2.01922336e-02   7.21225599e-02   1.12338142e-01 ...,   1.25014947e-01\n",
      "    2.59449761e-01   9.21289980e-01]\n",
      " [  6.78424679e-02   1.11762527e-01   8.22869604e-02 ...,   8.28531697e-01\n",
      "    1.67240900e-01   6.69106464e-02]]\n",
      "------\n"
     ]
    }
   ],
   "source": [
    "import itertools\n",
    "\n",
    "import tensorflow as tf\n",
    "from loss import cemse\n",
    "from keras import losses\n",
    "\n",
    "#data = list(itertools.islice(env.memory, 0, 73))\n",
    "data = env.memory\n",
    "\n",
    "sess = tf.Session()\n",
    "\n",
    "preds = best_player.predict(np.array([e[0] for e in data]))\n",
    "\n",
    "y_true = tf.constant(np.array([np.append(m[3], m[1]) for m in data]), dtype='float64')\n",
    "y_pred = tf.constant(preds, dtype='float64')\n",
    "\n",
    "#print('Y_TRUE')\n",
    "#print(sess.run(y_true))\n",
    "#print('Y_PRED')\n",
    "#print(sess.run(y_pred))\n",
    "\n",
    "v = tf.slice(y_pred, [0,0], [-1,1])\n",
    "z = tf.slice(y_true, [0,0], [-1,1])\n",
    "\n",
    "#print('TANH VALUE')\n",
    "#print(sess.run(tf.tanh(v)))\n",
    "#print('ACTUAL WINNER')\n",
    "#print(sess.run(z))\n",
    "\n",
    "p = tf.slice(y_pred, [0,1], [-1,-1])\n",
    "pi = tf.slice(y_true, [0,1], [-1,-1])\n",
    "\n",
    "#print('EXP LOGIT')\n",
    "#print(sess.run(tf.exp(p)/ tf.reduce_sum(tf.exp(p), axis = 1, keep_dims = True)))\n",
    "#print('ACTUAL PI')\n",
    "#print(sess.run(pi))\n",
    "\n",
    "\n",
    "\n",
    "loss1 = losses.mean_squared_error(z, tf.tanh(v))\n",
    "loss2 = tf.nn.softmax_cross_entropy_with_logits(labels = pi, logits = p)\n",
    "\n",
    "loss = cemse(y_true, y_pred)\n",
    "\n",
    "\n",
    "total_loss = tf.reduce_mean(loss)\n",
    "total_loss1 = tf.reduce_mean(loss1)\n",
    "total_loss2 = tf.reduce_mean(loss2)\n",
    "\n",
    "print('LOSSES')\n",
    "#print(sess.run(loss))\n",
    "#print(sess.run(loss1))\n",
    "#print(sess.run(loss2))\n",
    "\n",
    "print(sess.run(total_loss))\n",
    "print(sess.run(total_loss1))\n",
    "print(sess.run(total_loss2))\n",
    "\n",
    "print('------')\n",
    "\n",
    "print(sess.run(tf.reduce_mean(tf.abs(z - tf.tanh(v)))))\n",
    "print('PI')\n",
    "print(sess.run(pi))\n",
    "print('PROBS')\n",
    "print(sess.run(tf.exp(p)/ tf.reduce_sum(tf.exp(p), axis = 1, keep_dims = True) ))\n",
    "print('DIFF')\n",
    "print(sess.run(tf.abs(pi - (tf.exp(p)/ tf.reduce_sum(tf.exp(p), axis = 1, keep_dims = True) ))))\n",
    "\n",
    "print('------')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[array([0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0]), array([ 0.04958678,  0.        ,  0.03305785,  0.04958678,  0.08264463,\n",
      "        0.04958678,  0.        ,  0.        ,  0.73553719]), -1, 0]\n",
      "[ 0.11691926]\n",
      "[-1.]\n",
      "[ 0.09715707  0.11437535  0.12840817  0.09955741  0.11700293  0.1138888\n",
      "  0.1053149   0.11895457  0.10534081]\n",
      "[ 0.     0.212  0.     0.26   0.     0.     0.276  0.252  0.   ]\n"
     ]
    }
   ],
   "source": [
    "print(env.memory[269])\n",
    "print(sess.run(tf.tanh(v)[20]))\n",
    "print(sess.run(z[20]))\n",
    "print(sess.run((tf.exp(p)/ tf.reduce_sum(tf.exp(p), axis = 1, keep_dims = True))[20]))\n",
    "print(sess.run(pi[20]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAACbCAYAAAByBmCwAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAEvFJREFUeJzt3Xt01OWZB/DvM5NMEhJCQgIhhKs0\nK6ggyEVU1iKuCmiR7lov3R7Zapfqyq5Wux67e7bu2vW0e9Z66Vm3XaqsdI/rZaW2nK7X4gVdK4qX\nIiIKcg0QwiWJJCQhmXn2j4wnlPM+k/xmhpnxne/nHA4z7zPv7/fmzczDj5ln3ldUFURE9MUXyvYA\niIgoPZjQiYg8wYROROQJJnQiIk8woRMReYIJnYjIE0zoRESeYEInIvIEEzoRkScKUuksIvMBPAAg\nDOAhVf1RosdXVFZpbd0YZ+zosbCzfWjBEfN4PeGIHdPgP1po91YzVlRTZcYOS7WzvaQwGngMyVIV\nMybi/jawwu4zqH2/GesoHT7wgeWxku5WZ3ssXGj26QoNMmM9Mff1V0EoZvbp6Ha/roDMPj+Tkeg5\nHZEuZ3tU7Nd9TINfv1q/QwBoL6gwYyHjNZeszR++f1BVh/X3uKQTuoiEATwI4CIADQDeFpHVqrrJ\n6lNbNwYrn3rJGXt3p3tyrqp5xRxD8+DRZuxgt52ALcW3LjZj9bcsMWOPFXzT2X7GyJbAY0hWV9RO\nEkXhbmd7T8z+9U9df78Z2zhr2cAHlsfOaHzW2d5ZUWf22Vo8xYwd6ih1tleVtJt9PthjJ53JdZl7\nfiajM2pfsI0v3OFsbw0NNfsc7SkOPAbrdwgAb1fb+aKkwP0PTrLOnjR050Ael8pbLrMAbFXVbap6\nDMDjAC5P4XhERJSCVBJ6HYDdx91viLcREVEWnPQPRUVkqYisF5H1Lc0HT/bpiIjyVioJfQ+A49/E\nHhVv+wOqulxVZ6jqjIpK94eHRESUulQS+tsA6kVkvIhEAFwNYHV6hkVEREElXeWiqj0isgzA8+gt\nW1yhqh8me7zXX93rbB/7tfPMPk277cqOSSOCf4Jf8cDPzNiGaK0Zq2rLfvnXkIhd6WBVC4wqbDD7\n/EfF35ux82CXclGf1yr+zNl+9Jj9sqsr/syMjSo77GzviBaZfSbdPtOM4dEX7VgOmHLsTTO2peAs\nZ/uuQ+Vmn9FD7Lm1xCJ2Gemp4Y/M2C6cEvhc6ZBSHbqqPgPgmTSNhYiIUsBvihIReYIJnYjIE0zo\nRESeYEInIvJESh+KptOU6SOc7WfL62afx46en9YxbO8aa8ZmNT1txgaPucDZ3tZTkvKYBqpxzjwz\nVvGqew4/TfDz3tB1nxn7ANcNfGB57NzOF5zt7UNGmn0SVUdU6CFnewfs403+6V1mzK7RyA0Ng08z\nY3tbBjvbZwzdYvbZH60JPIaXI5eZsSrtMGMF0hP4XOnAK3QiIk8woRMReYIJnYjIE0zoRESeYEIn\nIvIEEzoRkSdypmzxhV++72yf8rfnmH2ua7vHjH2AbwUeQ0jsvRmjJWVmrPapu53tWxb/c+AxJKv2\n9TVmrMNYO2xMSaPZ5x8O3GDGrsSxAY8rnx2pdO+f++I+e5u5RIvKhTV4KdxLoYvNWC3s/XpzwYHO\nIWZs5mD3OoAb2ieZfYYW2wvYWapLj5qx0eFdZmxvLDt7/fAKnYjIE0zoRESeYEInIvIEEzoRkSeY\n0ImIPMGETkTkiZwpWzzvksnO9sob55h9tv/Xr+wDJrHYWWHI3hv0lchCM9Y9z70iWx2C72GYrKJQ\nlxmz9pwc3HnQ7PPea+6V/QDgynlfGvjA8tiOqHvlxLPq9pt9Eu0P+s6R053tNWV2+WFF8Re3xPR/\n16oZa5vlnouuHjH7DC0OPoao2sfbHXWXpQJAWLKzzzCv0ImIPMGETkTkCSZ0IiJPMKETEXmCCZ2I\nyBNM6EREnkipbFFEdgA4AiAKoEdVZyR7rN+9sNHZfsnD9iqCQ6LbzVgbRgceQ1e00IydMuSAGWvs\nqAx8rlwQC4XN2KKvn5Wgp70iIPU5RdwbFq9rtVdbrE1Qgiiz3aV62Pim2Wd0yV4zdrC7yozlgqsu\ntEsuY9rtbK8uPGz2OdQd/HU6/cPlZqxx2iIzdqCnOvC50iEddegXqKpd0ExERBnBt1yIiDyRakJX\nAC+IyDsistT1ABFZKiLrRWR9SzMv5ImITpZUE/ocVT0LwAIAN4nI+Sc+QFWXq+oMVZ1RUZmd95WI\niPJBSgldVffE/24C8DSAWekYFBERBZf0h6IiUgogpKpH4rcvBnBXsse7Ysk0Z/up0TfMPrGYXaWR\nzE82vLjZjL2xc6QZW1Szztm+HacGH0SSdrcPN2ODI53O9m3f+LbZ51s3LjBjH9bfOvCB5bFfbDrT\n2X7D2N+afTYnuCaasPklZ/uRBAvR7e2qMWORUBIr2GVQ/dF3zdiLnRc424+U2itwlUc6Ao9hbf0y\nM1bcaS/AVVJgL5Z3MqVS5VID4GkR+fw4/62qz6VlVEREFFjSCV1VtwFwX4IQEVHGsWyRiMgTTOhE\nRJ5gQici8gQTOhGRJ3JmT9G31rm/RVq/2F3OCACT8fu0jmF4xw4zVlVul3+t65ruPl5BW6pDGrDq\nEntRJ2vRsbYV/2f2eXCr/dSYm8G9Ur/Idmx3/05eHXuh2acG9u8xIlYp3CCzz9CIfby2nhIzlguO\nDrK/iDiq2P3aKr3tK2afznsT7EFs2NEUMWPzJuw0Y83dQwKfKx14hU5E5AkmdCIiTzChExF5ggmd\niMgTTOhERJ5gQici8kTOlC1OPN29v+GUmL3aYvi5J+wDLgq+8OOzzeeasekj7b0ZR7R94mzfgqmB\nx5Cs0lC7GeuKVjjbJw7aZvaZ85k9txtx88AHlsfOmuouXfvjsHvVRAD4BPZerq3R4KVwda3uvXoB\n4OPSmYGPl0mvHZpsxkZXuJ/v0ftX2QeMBR/DRRM+NWMNi75mxopX2b/jk4lX6EREnmBCJyLyBBM6\nEZEnmNCJiDzBhE5E5ImcqXK5oWa1s31/6RSzT+XFV6R1DGNutCsMdj36phnrLjWm0d5yMO261V5E\nyPJea70Ze3zz9WbsO+cEPlVeGjPsmLP9/g3nm30WzrQXdBuhDc72Bowx++woT1BplcHnZzJ2Ndqx\nM4e3BD5ed6wscJ89nSPMWOh/1ibomZ39WnmFTkTkCSZ0IiJPMKETEXmCCZ2IyBNM6EREnmBCJyLy\nRL9liyKyAsBlAJpU9Yx421AATwAYB2AHgCtVtTmVgcTWveps33rBArPPucc22wcsDj6GvcvfMWPh\nDrvfxFL3j94Jex/SdNvaYp+rpsy9r2St0Q4A3/lm8BIv+kMTyvY424edXW726Y6Fzdiu2Dhne0js\nVadqfn6bfbzrfmLGcsGlU+y6xWEdu5ztD38y2+wz9/Tge+HubbX3ax03NPf21h3IFfojAOaf0HYH\ngDWqWg9gTfw+ERFlUb8JXVXXAjh8QvPlAFbGb68EsDjN4yIiooCSfQ+9RlX3xW83Ahl8b4GIiJxS\n/lBUVRWAWnERWSoi60VkfUvzwVRPR0REhmQT+n4RqQWA+N9N1gNVdbmqzlDVGRWV1UmejoiI+pNs\nQl8NYEn89hIAv07PcIiIKFkDKVt8DMBcANUi0gDgTgA/AvCkiFwPYCeAK1MdSGj2XGd7acS9Yh0A\n7CybZh8wicXO6qvtFdze2Fppxhor3Suyif1OVNoN/2t7P1T85/PO5vaeIrPLqhe6zdi1C5PYnDEP\nHe5xP2c+aXLv8QoA9cPs5+Dpre7V/T6qmGP22fYX/27GCrK0IuBADYnab9E2DxrpbD99fHqfm+s3\ndJqxuvPtEtNIKDtz229CV9VrjNCFaR4LERGlgN8UJSLyBBM6EZEnmNCJiDzBhE5E5AkmdCIiT+TM\nJtF47w1nc+G8L5tdBmurGTuaxHKLmxrt0sT5E3eascav/qmzvejJNYHHkKzYit+asbCxG3Bbp72x\n9A2LEsxtTxJLWeahSMhd+jl5hPk9PHRG7d9JV1nwL+bVRuwVCw/05PYX/Zavm2jGLp3V7myfXvKB\n2WcXxgcew01/4t6YGwB2do6yO2apbJFX6EREnmBCJyLyBBM6EZEnmNCJiDzBhE5E5ImcqXJ5dtoP\nne3T7v6K2Ue+f09axzCtzq4IGHFokxk7uuo5Z3uXu7jkpGjvtqsjyiPuDVFHl9vbwK5aZ1dALJje\nNvCB5bFjsUJn+y3f3WD2ue9fp5ixxuJx7kCCgoqSbnvfWEhuV7n85exPzFhLqMrZHo0mSGlJrJXX\nErMr38ojR81Yor1hTyZeoRMReYIJnYjIE0zoRESeYEInIvIEEzoRkSeY0ImIPJEzZYuXbrnb2b7r\nzp+ZfaJp3rPzcFe5GduCS8xYQat7HDVlCUrG0iwak+B91C6tuvrsfWastWdw4HPlo5Dx/PzJPZPN\nPtEET+nyHy9zth+9+SGzzw5MMGMh5PbesNWbXzVj79ctdbYfKhpi9ikp6Ao8hoqQXdpbdXirGfuo\n/JzA50oHXqETEXmCCZ2IyBNM6EREnmBCJyLyBBM6EZEn+k3oIrJCRJpEZONxbf8oIntE5P34n4Un\nd5hERNSfgZQtPgLg3wD84oT2+1Q1bcsddkyb526PlZh9Hv6NezU7ALh2YfCSrHGRXWZscOEwM1aj\ne5ztDTom8BiSNal0mxlrjI5wtof/ZrHZ54eznzRjf3XNwMeVz4rC7jK5p35nr3K4cKa9kmXrrQ+6\nAwlW9SwtcK+0CQAd0SK7Yw54fuS3zVhPj7tMd9yQ9O6FeyBqv+5XN9WbsdnlLYHPlQ79XqGr6loA\nhzMwFiIiSkEq76EvE5EN8bdk7EWDiYgoI5JN6D8FMAHAVAD7APzYeqCILBWR9SKyvqX5YJKnIyKi\n/iSV0FV1v6pGVTUG4OcAZiV47HJVnaGqMyoqc3uHFCKiL7KkErqI1B5396sANlqPJSKizOi3ykVE\nHgMwF0C1iDQAuBPAXBGZit5d+nYAsD+OJiKijOg3oauqq0jt4XQPZNDG15ztXVPPMPtUVCYqQ7I3\ncLV82jXWjE0KbzZju0PjA58r3bpDCUrQjLK2z+591uxye5Fd/tWWYFNi6tMRdT8/F8xsT+p4W5vd\nJXR15Z+Zfd7eXWPGzhiZndK6gZpVZv/Hv0Hcr9WC711rH/AHdimupSxs55Frxn1sxrbj1MDnSgd+\nU5SIyBNM6EREnmBCJyLyBBM6EZEnmNCJiDyRM3uKYlits7krai/AdeopkQQHDF7lEhZ7Q8f9BaPM\nWLTH/e9iSDK3Z2NRNNHP616Zobb4gNnjByvtqpllXx/oqPJbadi9MNbzm9yLpQHAeX9kVxdNqHR/\n07ozar8Oaiu7zViu23TMrhSpKnYvYrb3+6vNPmXoDDyGcVueMWOvXHWvGat6643A50oHXqETEXmC\nCZ2IyBNM6EREnmBCJyLyBBM6EZEnmNCJiDwhqnapXtpPJnIAwM743WoA3PGiF+eiD+eiD+eiT77P\nxVhVtTc4jctoQv+DE4usV9UZWTl5juFc9OFc9OFc9OFcDAzfciEi8gQTOhGRJ7KZ0Jdn8dy5hnPR\nh3PRh3PRh3MxAFl7D52IiNKLb7kQEXkiKwldROaLyMcislVE7sjGGLJFRFaISJOIbDyubaiIvCgi\nW+J/u5dH9IyIjBaRl0Vkk4h8KCI3x9vzbj5EpFhE3hKR38fn4p/i7eNFZF38tfKEiCRaYtQbIhIW\nkfdE5Dfx+3k5D0FlPKGLSBjAgwAWADgNwDUiclqmx5FFjwCYf0LbHQDWqGo9gDXx+/mgB8Btqnoa\ngNkAboo/F/JxProAzFPVMwFMBTBfRGYD+BcA96nqlwA0A7g+i2PMpJsBfHTc/Xydh0CycYU+C8BW\nVd2mqscAPA7g8iyMIytUdS2Awyc0Xw5gZfz2SgCLMzqoLFHVfar6bvz2EfS+gOuQh/OhvT5f5Lsw\n/kcBzAPwVLw9L+ZCREYBuBTAQ/H7gjych2RkI6HXAdh93P2GeFs+q1HVffHbjQBqsjmYbBCRcQCm\nAViHPJ2P+NsM7wNoAvAigE8BtKhqT/wh+fJauR/A7QA+3yGmCvk5D4HxQ9Eco71lR3lVeiQiZQBW\nAbhFVT87PpZP86GqUVWdCmAUev8nOzHLQ8o4EbkMQJOqvpPtsXwRZWMLuj0ARh93f1S8LZ/tF5Fa\nVd0nIrXovULLCyJSiN5k/qiq/jLenLfzAQCq2iIiLwM4B0CFiBTEr07z4bVyHoBFIrIQQDGAcgAP\nIP/mISnZuEJ/G0B9/FPrCICrAdgbAeaH1QCWxG8vAfDrLI4lY+LvjT4M4CNVPX6DxrybDxEZJiIV\n8dslAC5C72cKLwO4Iv4w7+dCVb+nqqNUdRx6c8NLqvrnyLN5SFZWvlgU/9f3fgBhACtU9e6MDyJL\nROQxAHPRu3rcfgB3AvgVgCcBjEHvapRXquqJH5x6R0TmAHgNwAfoe7/079D7PnpezYeITEHvh31h\n9F5oPamqd4nIKegtHBgK4D0A31DVruyNNHNEZC6A76rqZfk8D0Hwm6JERJ7gh6JERJ5gQici8gQT\nOhGRJ5jQiYg8wYROROQJJnQiIk8woRMReYIJnYjIE/8PWkbtydMpLHYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x121031410>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAFEAAAD8CAYAAAAPDUgGAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAADHxJREFUeJztnWtwVOUZx///3SQEEzAQBMFwUUEt\n0ipKM6C2chGkWqUXh4qXWmvrdKqOTutQ7fihdvrBTmeq9oN2GMpIpwWqqNURFFPFgh21ooAIgly8\nQBQQSiBckuxmn37Yk91982azJ9mHgyHPb4bhPOc85z2HH++ey+553kMRgVEcsRO9AycDJlEBk6iA\nSVTAJCpgEhUwiQqYRAWKkkhyJsktJLeRvE9rp3oa7O4dC8k4gA8BTAewC8DbAOaIyKZ861RXnSoj\nhg3p1vai5tPP9mB/w0GGyS0pYju1ALaJyA4AILkEwCwAeSWOGDYEK//2WBGbjI4pN/08dG4xH+cz\nAOzMiXcF83odx/3EQvJ2kmtIrtl34ODx3twJoZiPcz2A4TlxTTDPQUTmAZgHAGNYLq/X3pFZNm3R\nT7xGd14824mHr33ay9nytZud+LzNT7ptnH+1t84Zu9504u3DJns5oz9anplmssVbno9ieuLbAMaQ\nPJNkGYDrATxfRHs9lm73RBFJkrwTwAoAcQALRGSj2p71IIr5OENElgNYXjDxJMfuWBQoqid2lYqx\nozBp8fxMnGhu9HKGv7nIiZNjLvByxqz6kxNfVXeNEy+7zj8ZJUd+xYlHr1/k5TSfnd2WlPTxlufD\neqICJlEBk6hApMdESgoliaOZuKFqlJczYHSrE++tPs/LOfTNWide+nX3dn017vTWGf/Uz5y44aPd\nXk6/By7PTKdicW95PqwnKmASFTCJCkR6TAQAYfb/rX+j930FWkvLnbj64A4vp7rANmqx2p/5vRud\n8LSOVjx2IDMZSyULbCWL9UQFTKICJlEBk6iASVTAJCpgEhUwiQpEfrEdS2W/YLjnxYu85Y/MXOPE\npfs/8xvZv9cJUyNGO3H8gLscAFJVg5w496K/I5hq7XR5LtYTFTCJCphEBUyiApGeWIQxJMoqMvFj\n4/1f5Y796z0nLq2d5OUsHz3XiSdWuc8M9O1f+PG90qZD3rymiuzJJxUvK9hGG9YTFTCJCphEBSL/\nta+05Uh2Rsx/mrfPpMucOFExwMupe9V9znH6tM/dhNUveeskrrjOiePH/GNi5ZFsu7Fks7c8H9YT\nFTCJCphEBSL/AiL3yYKm08/uVhsPTX/XbRPtnla43H/cOJ445sQt/Qd3ug2Jh1djPVEBk6hAQYkk\nF5DcS/L9nHkDSdaR3Br87V+H9CLC9MQnAMxsN+8+AK+IyBgArwRxr6WgRBFZBeB/7WbPArAwmF4I\n4DvK+9Wj6O4xcYiItN0m7AbQM6oejxNFn1gkXaaat1S1N5SldVfiHpJDASD42/9lKEBE5onIBBGZ\nMGjAqd3c3Jeb7kp8HsAtwfQtAJ7T2Z2eSZhLnMUA3gBwLsldJG8D8BCA6SS3ArgiiHstBe9tRGRO\nnkXTlPelx2J3LAqYRAVMogImUQGTqIBJVCDiX/sE8ZwBKmItR72ceMM+J/7u05d4Oc/c4H6zjfZ1\neLt3+Rs/vcYJU336+vvXmuxwuhDWExUwiQqYRAUifiqMaC3JPm2VO91G4pQqJ158l/8FUTNqvHkO\nA4Z2bwdzsF/7IsYkKmASFYj8CYhcyg75xzsmE2587IiXI0cOu/HQEW7cwfHs1mcuduK/TnnRy2kZ\n2r0nMqwnKmASFTCJCphEBU7oiWVR0v/5ZvbGe5142cV/8HJmvu8OHvTho0udeNyPZ3jrTL3y+07c\nsmmrl7Nq2E8z04fRr4M97hjriQqYRAVMogLdHvG9O4wfe470pMHK1276MNSI79YTFTCJCphEBUyi\nAiZRAZOogElUwCQqYBIVMIkKmEQFwjz4PpzkSpKbSG4keXcw3+r7AsL0xCSAX4rIWAATAdxBciys\nvi9DmNq+z0Xk3WC6EcAHSL8Vzer7Arp0TCQ5CsB4AG/B6vsyhJZIshLA0wDuERFnPJTO6vusti+A\nZCnSAv8uIs8Es0PV91ltHwCSBPAXAB+IyB9zFll9X0CYn0wvBXAzgA0k1wXzfo10Pd+TQa3fJwBm\n51n/pCdMbd/rAPL91mD1fbA7FhVMogImUQGTqIBJVMAkKmASFTCJCphEBUyiAiZRAZOogElUwCQq\nYBIVMIkKmEQFTKICJlEBk6iASVTAJCpgEhUwiQqYRAVMogImUYET+n7noxWnectPXfeyE9fX/sDL\niYs7QOSg/VuceGP/b3jrnH9wlRMnXlnm5ZTMmJUNulAHbj1RAZOogElUINJjYlOsApsrazPxuD0r\nvJxDF7iPPFa2HPByku1eG3x46WInPvsW9/32gD8I24Hr53o51Yc+ygYM37+sJypgEhUI8+B7Ocn/\nklwflKU9GMw/k+RbJLeR/AfJ8K/mPskI0xObAUwVkQsAXAhgJsmJAH4P4GERGQ3gAIDbjt9ufrkJ\n8+C7AGgbOrM0+CMApgK4IZi/EMBvADzeWVvlqSM45+g7mbiln38C6NPUjYKhObe6ccthLyUVL3Xi\n6oM7CjSqfLFNMh6UX+wFUAdgO4AGkcytwy6k6/16JaEkikiriFwIoAZALYDzwm7AytLaISINAFYC\nmASgimTb4aAGQH2edU76srSCx0SSpwFIiEgDyb4ApiN9UlkJ4DoASxCyLE1ApGLZYxOR8nLKd7vH\nquQAv3i15LB7AS5x9y0YP3xuorfOE7M3OHH8SIOX0/Le2mzQGP5TE+aOZSiAhSTjSPfcJ0XkBZKb\nACwh+TsAa5Gu/+uVhDk7v4d0jXP7+TuQPj72euyORQGTqEC0r1eCIJZKdJrTPHhkwXZaC7wqZP7N\n/oV0EhVu3KfCy8HknEvd+csL7kcb1hMVMIkKmEQFIv61jwCzRfxljfu8jB/9071qWjjjVb+ZdhfX\nLQOHOXHJ2tXeKk21VzpxxcfrvRyp6J/d09bOj925WE9UwCQqYBIViPiYKM6TBS2V1V7GvJu2O3Ez\nCl83tidx0eXevHiyyYmbas7ttA1p9yVuZ1hPVMAkKmASFTCJCphEBUyiAiZRAZOogElUwCQqYBIV\nMIkKmEQFTKICJlEBk6iASVTAJCpgEhUwiQqYRAVMogImUQGTqEBXXq8UJ7mW5AtBbLV9AV15AuJu\npN+U1vboVFtt3xKSf0a6tq/TsjSAkFj2ia7St+q8jGOXXuvE+8qHezmDF//WiftM+5YTbxgw1Vvn\n/MP/ceJPB1zo5Zy5543snqb88pB8hC1LqwFwNYD5QUyka/uWBin2yrkQPAJgLpCp3qlGyNo+pyyt\noZeWpZH8NoC9IvJOodyOcMrSqnppWRrSL/q6luRVAMqRPiY+iqC2L+iNeWv7egNhKqruB3A/AJCc\nDOBeEbmR5FPoYm0fJIV4IvuIW9Ol13gpR/tUOXHNvne9nPo5Dzjx6YfcwYXGNb7urVPy6WYnHpVo\n7nD/cgJ/eR6KuU78FYBfkNyG9DHSavvCICKvAXgtmLbavgC7Y1Eg2seN6V5sxzoocxi4YoE7Y4I/\neFrNF+5xUtoNBBRvavS3PdCtm76rbpKX8vhXs4MUMdnit5EH64kKmEQFTKICkQ9CmXv8irX6x53W\nS2YUvY3Wsr4Fcx6+ZoM3rwnjMtOpEG20YT1RAZOogElUwCQqYBIVMIkKmEQFTKICkV9s59JaUu7N\nO6X+AyduXO1/wXrs1vuduLJpvxMnSvwL5daY+08dWO9fbK8bmv2SuCnWwbg5ebCeqIBJVMAkKmAS\nFYh8cKHcb7Yrtvu/5ElFpRP3m+wPitF/67/ddcrcE1Synz9AR+ln7iBsiWFneTn3zs2O5Llz11Fv\neT6sJypgEhUwiQpEPrgQc16v1DRybGRbbg2xrWUPZn99nHKTvV4pUkyiAiZRAZOogElUwCQqYBIV\nMIkKmEQFTKICJlEBk6gApQvvMS56Y+QXAD4BMAiAP9x78Wi2O1JE/BdQd0CkEjMbJdeIyISe0m4h\n7OOsgElU4ERJnNfD2u2UE3JMPNmwj7MCkUokOZPklmDciPsU2/2Y5AaS60iu0Wo3NCISyR8AcaTf\nxnsWgDIA6wGMVWr7YwCDovq3tP8TZU+sBbBNRHaISAvSddKzItz+cSNKiWcA2JkTa74TWgC8TPId\nkrcrtRmaE/qQpyKXiUg9ycEA6khuFpFVUW08yp5YDyB3kBu1cSNEpD74ey+AZxFxMXuUEt8GMCYY\n2akMwPUAni+2UZIVJPu1TQOYAeD9YtvtCpF9nEUkSfJOACuQPlMvEJGNCk0PAfBserwjlABYJCIv\nKbQbGrtjUcDuWBQwiQqYRAVMogImUQGTqIBJVMAkKvB/CGsnBqEgq1oAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x11b3c1810>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "l1 = current_player.model.layers[0].get_weights()[0]\n",
    "plt.imshow(l1, cmap='coolwarm', interpolation='nearest')\n",
    "plt.show()\n",
    "\n",
    "\n",
    "l2 = current_player.model.layers[1].get_weights()[0]\n",
    "plt.imshow(l2, cmap='coolwarm', interpolation='nearest')\n",
    "plt.show()\n",
    "\n",
    "# l3 = current_player.model.layers[2].get_weights()[0]\n",
    "# plt.imshow(l3, cmap='coolwarm', interpolation='nearest')\n",
    "# plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "ename": "FailedPreconditionError",
     "evalue": "Attempting to use uninitialized value dense_4/kernel\n\t [[Node: dense_4/kernel/read = Identity[T=DT_FLOAT, _class=[\"loc:@dense_4/kernel\"], _device=\"/job:localhost/replica:0/task:0/device:CPU:0\"](dense_4/kernel)]]\n\nCaused by op u'dense_4/kernel/read', defined at:\n  File \"/usr/local/Cellar/python/2.7.13/Frameworks/Python.framework/Versions/2.7/lib/python2.7/runpy.py\", line 174, in _run_module_as_main\n    \"__main__\", fname, loader, pkg_name)\n  File \"/usr/local/Cellar/python/2.7.13/Frameworks/Python.framework/Versions/2.7/lib/python2.7/runpy.py\", line 72, in _run_code\n    exec code in run_globals\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/ipykernel_launcher.py\", line 16, in <module>\n    app.launch_new_instance()\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/traitlets/config/application.py\", line 658, in launch_instance\n    app.start()\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/ipykernel/kernelapp.py\", line 477, in start\n    ioloop.IOLoop.instance().start()\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/zmq/eventloop/ioloop.py\", line 177, in start\n    super(ZMQIOLoop, self).start()\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/tornado/ioloop.py\", line 888, in start\n    handler_func(fd_obj, events)\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/tornado/stack_context.py\", line 277, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py\", line 440, in _handle_events\n    self._handle_recv()\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py\", line 472, in _handle_recv\n    self._run_callback(callback, msg)\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py\", line 414, in _run_callback\n    callback(*args, **kwargs)\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/tornado/stack_context.py\", line 277, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/ipykernel/kernelbase.py\", line 283, in dispatcher\n    return self.dispatch_shell(stream, msg)\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/ipykernel/kernelbase.py\", line 235, in dispatch_shell\n    handler(stream, idents, msg)\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/ipykernel/kernelbase.py\", line 399, in execute_request\n    user_expressions, allow_stdin)\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/ipykernel/ipkernel.py\", line 196, in do_execute\n    res = shell.run_cell(code, store_history=store_history, silent=silent)\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/ipykernel/zmqshell.py\", line 533, in run_cell\n    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/IPython/core/interactiveshell.py\", line 2718, in run_cell\n    interactivity=interactivity, compiler=compiler, result=result)\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/IPython/core/interactiveshell.py\", line 2822, in run_ast_nodes\n    if self.run_code(code, result):\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/IPython/core/interactiveshell.py\", line 2882, in run_code\n    exec(code_obj, self.user_global_ns, self.user_ns)\n  File \"<ipython-input-3-ed082c4f0622>\", line 26, in <module>\n    best_player = agent.Agent(state_size, action_size, MCTSsimulations)\n  File \"agent.py\", line 33, in __init__\n    self.model = self._build_model(model)\n  File \"agent.py\", line 44, in _build_model\n    model.add(Dense(10, use_bias=True, bias_initializer='random_uniform', activation='relu', kernel_regularizer=regularizers.l2(0.00001), input_dim=self.state_size))\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/keras/models.py\", line 442, in add\n    layer(x)\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/keras/engine/topology.py\", line 575, in __call__\n    self.build(input_shapes[0])\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/keras/layers/core.py\", line 828, in build\n    constraint=self.kernel_constraint)\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/keras/legacy/interfaces.py\", line 87, in wrapper\n    return func(*args, **kwargs)\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/keras/engine/topology.py\", line 399, in add_weight\n    constraint=constraint)\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/keras/backend/tensorflow_backend.py\", line 316, in variable\n    v = tf.Variable(value, dtype=_convert_string_dtype(dtype), name=name)\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/tensorflow/python/ops/variables.py\", line 213, in __init__\n    constraint=constraint)\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/tensorflow/python/ops/variables.py\", line 360, in _init_from_args\n    self._snapshot = array_ops.identity(self._variable, name=\"read\")\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/tensorflow/python/ops/array_ops.py\", line 125, in identity\n    return gen_array_ops.identity(input, name=name)\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/tensorflow/python/ops/gen_array_ops.py\", line 2108, in identity\n    \"Identity\", input=input, name=name)\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/tensorflow/python/framework/op_def_library.py\", line 787, in _apply_op_helper\n    op_def=op_def)\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/tensorflow/python/framework/ops.py\", line 2991, in create_op\n    op_def=op_def)\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/tensorflow/python/framework/ops.py\", line 1479, in __init__\n    self._traceback = self._graph._extract_stack()  # pylint: disable=protected-access\n\nFailedPreconditionError (see above for traceback): Attempting to use uninitialized value dense_4/kernel\n\t [[Node: dense_4/kernel/read = Identity[T=DT_FLOAT, _class=[\"loc:@dense_4/kernel\"], _device=\"/job:localhost/replica:0/task:0/device:CPU:0\"](dense_4/kernel)]]\n",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFailedPreconditionError\u001b[0m                   Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-16-1e8675135d4f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m###### QUESTION 1 #########\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbest_player_version\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m     \u001b[0mpreds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbest_players\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mint\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m     \u001b[0;32mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mround\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpreds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0;32mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mround\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexp\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexp\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mpreds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/davidfoster/Git/AGI/scripts/agent.pyc\u001b[0m in \u001b[0;36mpredict\u001b[0;34m(self, inputToModel)\u001b[0m\n\u001b[1;32m    225\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputToModel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    226\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 227\u001b[0;31m                 \u001b[0mpreds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputToModel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    228\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    229\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mpreds\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/keras/models.pyc\u001b[0m in \u001b[0;36mpredict\u001b[0;34m(self, x, batch_size, verbose)\u001b[0m\n\u001b[1;32m    911\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuilt\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    912\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuild\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 913\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mverbose\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    914\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    915\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mpredict_on_batch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/keras/engine/training.pyc\u001b[0m in \u001b[0;36mpredict\u001b[0;34m(self, x, batch_size, verbose, steps)\u001b[0m\n\u001b[1;32m   1711\u001b[0m         \u001b[0mf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict_function\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1712\u001b[0m         return self._predict_loop(f, ins, batch_size=batch_size,\n\u001b[0;32m-> 1713\u001b[0;31m                                   verbose=verbose, steps=steps)\n\u001b[0m\u001b[1;32m   1714\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1715\u001b[0m     def train_on_batch(self, x, y,\n",
      "\u001b[0;32m/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/keras/engine/training.pyc\u001b[0m in \u001b[0;36m_predict_loop\u001b[0;34m(self, f, ins, batch_size, verbose, steps)\u001b[0m\n\u001b[1;32m   1267\u001b[0m                 \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1268\u001b[0m                     \u001b[0mins_batch\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_slice_arrays\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_ids\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1269\u001b[0;31m                 \u001b[0mbatch_outs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1270\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_outs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1271\u001b[0m                     \u001b[0mbatch_outs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mbatch_outs\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/keras/backend/tensorflow_backend.pyc\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2271\u001b[0m         updated = session.run(self.outputs + [self.updates_op],\n\u001b[1;32m   2272\u001b[0m                               \u001b[0mfeed_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2273\u001b[0;31m                               **self.session_kwargs)\n\u001b[0m\u001b[1;32m   2274\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mupdated\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2275\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    887\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    888\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 889\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    890\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    891\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1118\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1119\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m-> 1120\u001b[0;31m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[1;32m   1121\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1122\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1315\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1316\u001b[0m       return self._do_call(_run_fn, self._session, feeds, fetches, targets,\n\u001b[0;32m-> 1317\u001b[0;31m                            options, run_metadata)\n\u001b[0m\u001b[1;32m   1318\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1319\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1334\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1335\u001b[0m           \u001b[0;32mpass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1336\u001b[0;31m       \u001b[0;32mraise\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnode_def\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1337\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1338\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_extend_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mFailedPreconditionError\u001b[0m: Attempting to use uninitialized value dense_4/kernel\n\t [[Node: dense_4/kernel/read = Identity[T=DT_FLOAT, _class=[\"loc:@dense_4/kernel\"], _device=\"/job:localhost/replica:0/task:0/device:CPU:0\"](dense_4/kernel)]]\n\nCaused by op u'dense_4/kernel/read', defined at:\n  File \"/usr/local/Cellar/python/2.7.13/Frameworks/Python.framework/Versions/2.7/lib/python2.7/runpy.py\", line 174, in _run_module_as_main\n    \"__main__\", fname, loader, pkg_name)\n  File \"/usr/local/Cellar/python/2.7.13/Frameworks/Python.framework/Versions/2.7/lib/python2.7/runpy.py\", line 72, in _run_code\n    exec code in run_globals\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/ipykernel_launcher.py\", line 16, in <module>\n    app.launch_new_instance()\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/traitlets/config/application.py\", line 658, in launch_instance\n    app.start()\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/ipykernel/kernelapp.py\", line 477, in start\n    ioloop.IOLoop.instance().start()\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/zmq/eventloop/ioloop.py\", line 177, in start\n    super(ZMQIOLoop, self).start()\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/tornado/ioloop.py\", line 888, in start\n    handler_func(fd_obj, events)\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/tornado/stack_context.py\", line 277, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py\", line 440, in _handle_events\n    self._handle_recv()\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py\", line 472, in _handle_recv\n    self._run_callback(callback, msg)\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py\", line 414, in _run_callback\n    callback(*args, **kwargs)\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/tornado/stack_context.py\", line 277, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/ipykernel/kernelbase.py\", line 283, in dispatcher\n    return self.dispatch_shell(stream, msg)\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/ipykernel/kernelbase.py\", line 235, in dispatch_shell\n    handler(stream, idents, msg)\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/ipykernel/kernelbase.py\", line 399, in execute_request\n    user_expressions, allow_stdin)\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/ipykernel/ipkernel.py\", line 196, in do_execute\n    res = shell.run_cell(code, store_history=store_history, silent=silent)\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/ipykernel/zmqshell.py\", line 533, in run_cell\n    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/IPython/core/interactiveshell.py\", line 2718, in run_cell\n    interactivity=interactivity, compiler=compiler, result=result)\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/IPython/core/interactiveshell.py\", line 2822, in run_ast_nodes\n    if self.run_code(code, result):\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/IPython/core/interactiveshell.py\", line 2882, in run_code\n    exec(code_obj, self.user_global_ns, self.user_ns)\n  File \"<ipython-input-3-ed082c4f0622>\", line 26, in <module>\n    best_player = agent.Agent(state_size, action_size, MCTSsimulations)\n  File \"agent.py\", line 33, in __init__\n    self.model = self._build_model(model)\n  File \"agent.py\", line 44, in _build_model\n    model.add(Dense(10, use_bias=True, bias_initializer='random_uniform', activation='relu', kernel_regularizer=regularizers.l2(0.00001), input_dim=self.state_size))\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/keras/models.py\", line 442, in add\n    layer(x)\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/keras/engine/topology.py\", line 575, in __call__\n    self.build(input_shapes[0])\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/keras/layers/core.py\", line 828, in build\n    constraint=self.kernel_constraint)\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/keras/legacy/interfaces.py\", line 87, in wrapper\n    return func(*args, **kwargs)\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/keras/engine/topology.py\", line 399, in add_weight\n    constraint=constraint)\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/keras/backend/tensorflow_backend.py\", line 316, in variable\n    v = tf.Variable(value, dtype=_convert_string_dtype(dtype), name=name)\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/tensorflow/python/ops/variables.py\", line 213, in __init__\n    constraint=constraint)\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/tensorflow/python/ops/variables.py\", line 360, in _init_from_args\n    self._snapshot = array_ops.identity(self._variable, name=\"read\")\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/tensorflow/python/ops/array_ops.py\", line 125, in identity\n    return gen_array_ops.identity(input, name=name)\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/tensorflow/python/ops/gen_array_ops.py\", line 2108, in identity\n    \"Identity\", input=input, name=name)\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/tensorflow/python/framework/op_def_library.py\", line 787, in _apply_op_helper\n    op_def=op_def)\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/tensorflow/python/framework/ops.py\", line 2991, in create_op\n    op_def=op_def)\n  File \"/Users/davidfoster/.virtualenvs/deepreinforcement/lib/python2.7/site-packages/tensorflow/python/framework/ops.py\", line 1479, in __init__\n    self._traceback = self._graph._extract_stack()  # pylint: disable=protected-access\n\nFailedPreconditionError (see above for traceback): Attempting to use uninitialized value dense_4/kernel\n\t [[Node: dense_4/kernel/read = Identity[T=DT_FLOAT, _class=[\"loc:@dense_4/kernel\"], _device=\"/job:localhost/replica:0/task:0/device:CPU:0\"](dense_4/kernel)]]\n"
     ]
    }
   ],
   "source": [
    "###### QUESTION 1 #########\n",
    "for i in range(best_player_version+1):\n",
    "    preds = best_players[i].predict(np.array([[1,1,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0]], dtype=np.int))\n",
    "    print(round(preds[0],2))\n",
    "    print([round(np.exp(x) / (1 + np.exp(x)),2) for x in preds[1:]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'Sequential' object has no attribute 'size'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-13-c60f811ca67e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mbest_players\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m: 'Sequential' object has no attribute 'size'"
     ]
    }
   ],
   "source": [
    "best_players[i].model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[array([0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0]),\n",
       "  array([ 0.82608696,  0.        ,  0.        ,  0.06521739,  0.        ,\n",
       "          0.02173913,  0.        ,  0.06521739,  0.02173913]),\n",
       "  1,\n",
       "  1],\n",
       " [array([1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1]),\n",
       "  array([ 0.        ,  0.18181818,  0.        ,  0.16666667,  0.        ,\n",
       "          0.        ,  0.46969697,  0.18181818,  0.        ]),\n",
       "  -1,\n",
       "  -1]]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import itertools\n",
    "output = list(itertools.islice(env.memory, 71, 73))\n",
    "output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from loss import cemse\n",
    "from keras import losses\n",
    "\n",
    "data = list(itertools.islice(env.memory, 0, 73))\n",
    "# data = env.memory\n",
    "\n",
    "sess = tf.Session()\n",
    "\n",
    "preds = current_player.predict(np.array([e[0] for e in data]))\n",
    "\n",
    "y_true = tf.constant(np.array([np.append(m[3], m[1]) for m in data]), dtype='float64')\n",
    "y_pred = tf.constant(preds, dtype='float64')\n",
    "\n",
    "#print('Y_TRUE')\n",
    "#print(sess.run(y_true))\n",
    "#print('Y_PRED')\n",
    "#print(sess.run(y_pred))\n",
    "\n",
    "v = tf.slice(y_pred, [0,0], [-1,1])\n",
    "z = tf.slice(y_true, [0,0], [-1,1])\n",
    "\n",
    "#print('TANH VALUE')\n",
    "#print(sess.run(tf.tanh(v)))\n",
    "#print('ACTUAL WINNER')\n",
    "#print(sess.run(z))\n",
    "\n",
    "p = tf.slice(y_pred, [0,1], [-1,-1])\n",
    "pi = tf.slice(y_true, [0,1], [-1,-1])\n",
    "\n",
    "#print('EXP LOGIT')\n",
    "#print(sess.run(tf.exp(p)/ tf.reduce_sum(tf.exp(p), axis = 1, keep_dims = True)))\n",
    "#print('ACTUAL PI')\n",
    "#print(sess.run(pi))\n",
    "\n",
    "\n",
    "\n",
    "loss1 = losses.mean_squared_error(z, tf.tanh(v))\n",
    "loss2 = tf.nn.softmax_cross_entropy_with_logits(labels = pi, logits = p)\n",
    "\n",
    "loss = cemse(y_true, y_pred)\n",
    "\n",
    "\n",
    "total_loss = tf.reduce_mean(loss)\n",
    "total_loss1 = tf.reduce_mean(loss1)\n",
    "total_loss2 = tf.reduce_mean(loss2)\n",
    "\n",
    "print('LOSSES')\n",
    "#print(sess.run(loss))\n",
    "#print(sess.run(loss1))\n",
    "#print(sess.run(loss2))\n",
    "\n",
    "print(sess.run(total_loss))\n",
    "print(sess.run(total_loss1))\n",
    "print(sess.run(total_loss2))\n",
    "\n",
    "print('------')\n",
    "\n",
    "print(sess.run(tf.reduce_mean(tf.abs(z - tf.tanh(v)))))\n",
    "print(sess.run(tf.reduce_mean(tf.abs(pi - p))))\n",
    "\n",
    "#print(preds[2])\n",
    "#print(np.array([np.append(m[3], m[1]) for m in env.memory])[2])\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "deepreinforcement",
   "language": "python",
   "name": "deepreinforcement"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
